mirror of
https://github.com/TwilitRealm/dusklight
synced 2026-08-21 21:40:32 -04:00
Merge branch 'master' into master
This commit is contained in:
Darek Konopka
@@ -1,5 +1,8 @@
|
||||
import os
|
||||
import sys
|
||||
import libarc
|
||||
from pathlib import Path
|
||||
import libyaz0
|
||||
|
||||
"""
|
||||
Extracts the game assets and stores them in the game folder
|
||||
@@ -127,6 +130,41 @@ def writeFolder(parsedFstBin, i):
|
||||
Use the parsed fst.bin contents to write assets to file
|
||||
"""
|
||||
|
||||
convertDefinitions = [
|
||||
{
|
||||
"extension": ".arc",
|
||||
"function": libarc.extract_to_directory,
|
||||
"exceptions": ["archive/dat/speakerse.arc"],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def writeFile(name, data):
|
||||
if data[0:4] == bytes("Yaz0", "ascii"):
|
||||
splitName = os.path.splitext(name)
|
||||
name = splitName[0] + ".c" + splitName[1]
|
||||
data = libyaz0.decompress(data)
|
||||
|
||||
extractDef = None
|
||||
splitName = os.path.splitext(name)
|
||||
ext = splitName[1]
|
||||
for extractData in convertDefinitions:
|
||||
if ext == extractData["extension"]:
|
||||
extractDef = extractData
|
||||
if extractData["exceptions"] != None:
|
||||
for exception in extractData["exceptions"]:
|
||||
if str(name) == exception:
|
||||
extractDef = None
|
||||
break
|
||||
|
||||
if extractDef == None:
|
||||
file = open(name, "wb")
|
||||
file.write(data)
|
||||
file.close()
|
||||
else:
|
||||
name = extractDef["function"](name, data, writeFile)
|
||||
return name
|
||||
|
||||
|
||||
def writeAssets(parsedFstBin, handler):
|
||||
# Write the folder structure and files to disc
|
||||
@@ -145,10 +183,14 @@ def writeAssets(parsedFstBin, handler):
|
||||
)
|
||||
else:
|
||||
handler.seek(i["fileOffset"])
|
||||
with open(
|
||||
(folderStack[-1]["folderName"] + i["fileName"]), "wb"
|
||||
) as currentFile:
|
||||
currentFile.write(bytearray(handler.read(i["fileSize"])))
|
||||
writeFile(
|
||||
folderStack[-1]["folderName"] + i["fileName"],
|
||||
bytearray(handler.read(i["fileSize"])),
|
||||
)
|
||||
# with open(
|
||||
# (folderStack[-1]["folderName"] + i["fileName"]), "wb"
|
||||
# ) as currentFile:
|
||||
# currentFile.write(bytearray(handler.read(i["fileSize"])))
|
||||
|
||||
while folderStack[-1]["lastEntryNumber"] == j + 1:
|
||||
folderStack.pop()
|
||||
|
||||
+378
-27
@@ -5,7 +5,10 @@ Simple library for reading and paring rarc files.
|
||||
"""
|
||||
|
||||
import struct
|
||||
import os
|
||||
import ctypes
|
||||
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Dict
|
||||
|
||||
@@ -16,15 +19,17 @@ from typing import List, Dict
|
||||
|
||||
NODE_SIZE = 0x10
|
||||
DIRECTORY_SIZE = 0x14
|
||||
ROOT = struct.unpack('>I', "ROOT".encode('ascii'))[0]
|
||||
ROOT = struct.unpack(">I", "ROOT".encode("ascii"))[0]
|
||||
|
||||
|
||||
def chunks(lst, n):
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
yield lst[i : i + n]
|
||||
|
||||
|
||||
@dataclass
|
||||
class StringTable:
|
||||
""" RARC String Table """
|
||||
"""RARC String Table"""
|
||||
|
||||
strings: Dict[int, str] = field(default_factory=dict)
|
||||
|
||||
@@ -34,7 +39,7 @@ class StringTable:
|
||||
|
||||
@dataclass
|
||||
class Directory:
|
||||
""" RARC Directory """
|
||||
"""RARC Directory"""
|
||||
|
||||
index: int
|
||||
name_hash: int
|
||||
@@ -50,17 +55,17 @@ class Directory:
|
||||
|
||||
@dataclass
|
||||
class File(Directory):
|
||||
""" RARC File """
|
||||
"""RARC File"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Folder(Directory):
|
||||
""" RARC Folder """
|
||||
"""RARC Folder"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Node:
|
||||
""" RARC Node """
|
||||
"""RARC Node"""
|
||||
|
||||
identifier: int
|
||||
name_offset: int
|
||||
@@ -69,11 +74,14 @@ class Node:
|
||||
directory_index: int
|
||||
|
||||
name: str = None
|
||||
parent = None
|
||||
rarc: "RARC" = field(default=None, repr=False)
|
||||
|
||||
def files_and_folders(self, depth):
|
||||
""" Generator for eacg file and directory of this node """
|
||||
for directory in self.rarc._directories[self.directory_index:][:self.directory_count]:
|
||||
"""Generator for eacg file and directory of this node"""
|
||||
for directory in self.rarc._directories[self.directory_index :][
|
||||
: self.directory_count
|
||||
]:
|
||||
yield depth, directory
|
||||
if isinstance(directory, Folder):
|
||||
if directory.data_offset < len(self.rarc._nodes):
|
||||
@@ -85,7 +93,7 @@ class Node:
|
||||
|
||||
@dataclass
|
||||
class RARC:
|
||||
"""
|
||||
"""
|
||||
RARC - Archive of files and folder
|
||||
"""
|
||||
|
||||
@@ -95,8 +103,8 @@ class RARC:
|
||||
header_length: int
|
||||
file_offset: int
|
||||
file_data_length: int
|
||||
file_data_length2: int
|
||||
unknown0: int
|
||||
file_data_mmem: int
|
||||
file_data_amem: int
|
||||
unknown1: int
|
||||
|
||||
# info block
|
||||
@@ -117,29 +125,29 @@ class RARC:
|
||||
|
||||
@property
|
||||
def files_and_folders(self):
|
||||
""" Generator for each file and directory """
|
||||
"""Generator for each file and directory"""
|
||||
yield from self._root.files_and_folders(0)
|
||||
|
||||
|
||||
def read_string_table(rarc, data):
|
||||
buffer = data[rarc.string_table_offset:][:rarc.string_table_length]
|
||||
buffer = data[rarc.string_table_offset :][: rarc.string_table_length]
|
||||
rarc.string_table = StringTable()
|
||||
|
||||
offset = 0
|
||||
for string in buffer.decode('ascii').split('\0'):
|
||||
for string in str(buffer, "shift-jis").split("\0"):
|
||||
rarc.string_table.strings[offset] = string
|
||||
offset += len(string) + 1
|
||||
offset += len(bytearray(string, "shift-jis")) + 1
|
||||
|
||||
|
||||
def read_node(rarc, buffer):
|
||||
node = Node(*struct.unpack('>IIHHI', buffer))
|
||||
node = Node(*struct.unpack(">IIHHI", buffer))
|
||||
node.name = rarc.string_table.get(node.name_offset)
|
||||
node.rarc = rarc
|
||||
return node
|
||||
|
||||
|
||||
def read_nodes(rarc, data):
|
||||
buffer = data[rarc.node_offset:][:rarc.node_count * NODE_SIZE]
|
||||
buffer = data[rarc.node_offset :][: rarc.node_count * NODE_SIZE]
|
||||
rarc._nodes = []
|
||||
for node_buffer in chunks(buffer, NODE_SIZE):
|
||||
node = read_node(rarc, node_buffer)
|
||||
@@ -149,38 +157,381 @@ def read_nodes(rarc, data):
|
||||
|
||||
|
||||
def read_directory(rarc, buffer, file_data):
|
||||
header = struct.unpack('>HHHHIII', buffer)
|
||||
header = struct.unpack(">HHHHIII", buffer)
|
||||
if header[0] == 0xFFFF:
|
||||
directory = Folder(*header)
|
||||
else:
|
||||
directory = File(*header)
|
||||
directory.data = file_data[directory.data_offset:][:directory.data_length]
|
||||
directory.data = file_data[directory.data_offset :][: directory.data_length]
|
||||
directory.name = rarc.string_table.get(directory.name_offset)
|
||||
directory.rarc = rarc
|
||||
return directory
|
||||
|
||||
|
||||
def read_directories(rarc, data, file_data):
|
||||
buffer = data[rarc.directory_offset:][:rarc.directory_count * DIRECTORY_SIZE]
|
||||
buffer = data[rarc.directory_offset :][: rarc.directory_count * DIRECTORY_SIZE]
|
||||
rarc._directories = []
|
||||
for directory_buffer in chunks(buffer, DIRECTORY_SIZE):
|
||||
rarc._directories.append(read_directory(
|
||||
rarc, directory_buffer, file_data))
|
||||
rarc._directories.append(read_directory(rarc, directory_buffer, file_data))
|
||||
|
||||
|
||||
def read(buffer) -> RARC:
|
||||
""" Read and parse RARC from buffer. """
|
||||
"""Read and parse RARC from buffer."""
|
||||
|
||||
# TODO: Add error checking
|
||||
header = struct.unpack('>IIIIIIII', buffer[:32])
|
||||
info = struct.unpack('>IIIIIIHHI', buffer[32:][:32])
|
||||
header = struct.unpack(">IIIIIIII", buffer[:32])
|
||||
info = struct.unpack(">IIIIIIHHI", buffer[32:][:32])
|
||||
rarc = RARC(*header, *info)
|
||||
|
||||
data = buffer[32:]
|
||||
file_data = data[rarc.file_offset:][:rarc.file_length]
|
||||
file_data = data[rarc.file_offset :][: rarc.file_length]
|
||||
|
||||
read_string_table(rarc, data)
|
||||
read_nodes(rarc, data)
|
||||
read_directories(rarc, data, file_data)
|
||||
|
||||
return rarc
|
||||
|
||||
|
||||
def extract_node(node, arcData, write_function, parentDir, dirNames) -> str:
|
||||
os.mkdir(Path(parentDir) / node.name)
|
||||
for i in range(node.directory_index, node.directory_count + node.directory_index):
|
||||
dir = arcData._directories[i]
|
||||
dirNames[i] = str(Path(parentDir) / Path(node.name)) + "/" + dir.name
|
||||
if type(dir) == Folder and dir.name != "." and dir.name != "..":
|
||||
for j, node2 in enumerate(arcData._nodes):
|
||||
if dir.data_offset == j:
|
||||
dirNames = extract_node(
|
||||
node2,
|
||||
arcData,
|
||||
write_function,
|
||||
Path(parentDir) / node.name,
|
||||
dirNames,
|
||||
)
|
||||
break
|
||||
elif type(dir) == File:
|
||||
dirNames[i] = write_function(
|
||||
Path(parentDir) / Path(node.name) / dir.name, dir.data
|
||||
)
|
||||
|
||||
return dirNames
|
||||
|
||||
|
||||
def extract_to_directory(directory, data, write_function):
|
||||
print("Extracting " + str(directory))
|
||||
os.mkdir(directory)
|
||||
arcData = read(data)
|
||||
cwd = os.getcwd()
|
||||
os.chdir(directory)
|
||||
dirNames = extract_node(
|
||||
arcData._root, arcData, write_function, "./", [None] * len(arcData._directories)
|
||||
)
|
||||
|
||||
files_data = ""
|
||||
for i, dir in enumerate(arcData._directories):
|
||||
directoryIndicator = ""
|
||||
specialType = ""
|
||||
indexToUse = str(dir.index).zfill(len(str(len(arcData._directories))))
|
||||
if type(dir) == Folder:
|
||||
directoryIndicator = "/"
|
||||
indexToUse = "Folder"
|
||||
if dir.type != 0x200 and dir.type != 0x1100 and dir.type != 0x9500:
|
||||
specialType = ":" + hex(dir.type)
|
||||
files_data = (
|
||||
files_data
|
||||
+ indexToUse
|
||||
+ ":"
|
||||
+ str(dirNames[i])
|
||||
+ directoryIndicator
|
||||
+ specialType
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
fileDataLines = files_data.splitlines()
|
||||
# fileDataLines.sort(key=lambda x : int(x.split(":")[0]))
|
||||
filesFile = open("_files.txt", "w")
|
||||
for line in fileDataLines:
|
||||
filesFile.write(line + "\n")
|
||||
os.chdir(cwd)
|
||||
return directory
|
||||
|
||||
|
||||
def computeHash(string):
|
||||
hash = 0
|
||||
for char in string:
|
||||
hash = hash * 3
|
||||
hash = hash + ord(char)
|
||||
|
||||
hash = ctypes.c_ushort(hash)
|
||||
hash = hash.value
|
||||
return hash
|
||||
|
||||
|
||||
def getNodeIdent(fullName):
|
||||
if len(fullName) < 4:
|
||||
fullName = fullName.upper()
|
||||
for i in range(4 - len(fullName)):
|
||||
fullName = fullName + " "
|
||||
else:
|
||||
fullName = fullName.upper()[:4]
|
||||
return struct.unpack(">I", fullName.encode("ascii"))[0]
|
||||
|
||||
|
||||
def parseDirForPack(
|
||||
fileDataLines, path, convertFunction, nodes, dirs, currentNode, stringTable, data
|
||||
):
|
||||
for i in range(
|
||||
currentNode.directory_index,
|
||||
currentNode.directory_count + currentNode.directory_index,
|
||||
):
|
||||
currentLine = fileDataLines[i].split(":")
|
||||
dirId = currentLine[0]
|
||||
if dirId == "Folder":
|
||||
dirId = 0xFFFF
|
||||
else:
|
||||
dirId = int(dirId)
|
||||
currentLineName = currentLine[1]
|
||||
specialDirType = 0
|
||||
if len(currentLine) > 2:
|
||||
specialDirType = int(currentLine[2], 16)
|
||||
if currentLineName[-1] == "/":
|
||||
currentLineName = currentLineName[0:-1]
|
||||
dirName = currentLineName.split("/")[-1]
|
||||
if (
|
||||
dirName == "."
|
||||
or dirName == ".."
|
||||
or (
|
||||
os.path.isdir(path / currentLineName)
|
||||
and len(os.path.splitext(dirName)[1]) == 0
|
||||
)
|
||||
):
|
||||
stringTableOffset = 0
|
||||
nodeIndex = nodes.index(currentNode)
|
||||
if dirName == "..":
|
||||
if currentNode.parent == None:
|
||||
nodeIndex = 0xFFFFFFFF
|
||||
else:
|
||||
nodeIndex = nodes.index(currentNode.parent)
|
||||
stringTableOffset = 2
|
||||
if dirName != "." and dirName != "..":
|
||||
stringTableOffset = len(bytearray(stringTable, "shift-jis"))
|
||||
stringTable = stringTable + dirName + "\0"
|
||||
dirsInCurrentDir = []
|
||||
for j, line in enumerate(fileDataLines):
|
||||
split = line.split(":")[1].split("/")
|
||||
if split[-1] == "":
|
||||
split.pop()
|
||||
if currentLineName == "/".join(split[0:-1]):
|
||||
dirsInCurrentDir.append(j)
|
||||
newNode = Node(
|
||||
getNodeIdent(dirName),
|
||||
stringTableOffset,
|
||||
computeHash(dirName),
|
||||
len(dirsInCurrentDir),
|
||||
dirsInCurrentDir[0],
|
||||
dirName,
|
||||
)
|
||||
newNode.parent = currentNode
|
||||
nodes.append(newNode)
|
||||
nodeIndex = len(nodes) - 1
|
||||
stringTable, nodes, dirs, data = parseDirForPack(
|
||||
fileDataLines,
|
||||
path,
|
||||
convertFunction,
|
||||
nodes,
|
||||
dirs,
|
||||
newNode,
|
||||
stringTable,
|
||||
data,
|
||||
)
|
||||
dirs[i] = Folder(
|
||||
dirId,
|
||||
computeHash(dirName),
|
||||
0x200,
|
||||
stringTableOffset,
|
||||
nodeIndex,
|
||||
16,
|
||||
0,
|
||||
dirName,
|
||||
)
|
||||
else:
|
||||
realFileName, fileData = convertFunction(currentLineName, path, None, True)
|
||||
realFileName = os.path.basename(realFileName)
|
||||
stringTableOffset = len(bytearray(stringTable, "shift-jis"))
|
||||
stringTable = stringTable + realFileName + "\0"
|
||||
fileType = 0x1100
|
||||
if fileData[:4] == bytearray("Yaz0", "utf-8"):
|
||||
fileType = 0x9500
|
||||
if specialDirType != 0:
|
||||
fileType = specialDirType
|
||||
dirs[i] = File(
|
||||
dirId,
|
||||
computeHash(realFileName),
|
||||
fileType,
|
||||
stringTableOffset,
|
||||
len(data),
|
||||
len(fileData),
|
||||
0,
|
||||
realFileName,
|
||||
)
|
||||
data = data + fileData
|
||||
fileEndPadding = 0x20 - (len(data) % 0x20)
|
||||
if fileEndPadding == 0x20:
|
||||
fileEndPadding = 0
|
||||
data = data + bytearray(fileEndPadding)
|
||||
return stringTable, nodes, dirs, data
|
||||
|
||||
|
||||
def convert_dir_to_arc(sourceDir, convertFunction):
|
||||
# print("Converting "+str(sourceDir))
|
||||
fileData = open(sourceDir / "_files.txt", "r").read()
|
||||
fileDataLinesFull = fileData.splitlines()
|
||||
# fileDataLinesFull.sort(key=lambda x : int(x.split(":")[0]))
|
||||
|
||||
fileDataLines = []
|
||||
for line in fileDataLinesFull:
|
||||
# fileDataLines.append(":".join(line.split(":")[1:])) #this should map directory ids to their index directly
|
||||
fileDataLines.append(line)
|
||||
|
||||
rootName = fileDataLines[0].split(":")[1].split("/")[0]
|
||||
nodes = []
|
||||
dirs = [None] * len(fileDataLines)
|
||||
stringTable = ".\0..\0"
|
||||
nodes.append(
|
||||
Node(
|
||||
getNodeIdent("ROOT"),
|
||||
len(stringTable),
|
||||
computeHash(rootName),
|
||||
len(os.listdir(sourceDir / rootName)) + 2,
|
||||
0,
|
||||
rootName,
|
||||
)
|
||||
)
|
||||
stringTable = stringTable + rootName + "\0"
|
||||
data = bytearray(0)
|
||||
|
||||
stringTable, nodes, dirs, data = parseDirForPack(
|
||||
fileDataLines,
|
||||
sourceDir,
|
||||
convertFunction,
|
||||
nodes,
|
||||
dirs,
|
||||
nodes[0],
|
||||
stringTable,
|
||||
data,
|
||||
)
|
||||
|
||||
dirOffset = 32 + (len(nodes) * 16)
|
||||
dirOffsetPadding = 0x20 - (dirOffset % 0x20)
|
||||
if dirOffsetPadding == 0x20:
|
||||
dirOffsetPadding = 0
|
||||
dirOffset = dirOffset + dirOffsetPadding
|
||||
stringTableOffset = dirOffset + (len(dirs) * 20)
|
||||
stringTablePadding = 0x20 - (stringTableOffset % 0x20)
|
||||
stringTableOffset = stringTableOffset + stringTablePadding
|
||||
stringTableLen = len(bytearray(stringTable, "shift-jis"))
|
||||
fileOffset = stringTableOffset + stringTableLen
|
||||
fileOffsetPadding = 0x20 - (fileOffset % 0x20)
|
||||
if fileOffsetPadding == 0x20:
|
||||
fileOffsetPadding = 0
|
||||
fileOffset = fileOffset + fileOffsetPadding
|
||||
|
||||
fileLength = fileOffset + len(data)
|
||||
|
||||
mMemLength = len(data)
|
||||
aMemLength = 0
|
||||
|
||||
fileCount = len(dirs)
|
||||
folderCount = 0
|
||||
for dir in dirs:
|
||||
if type(dir) == Folder:
|
||||
folderCount = folderCount + 1
|
||||
if aMemLength == 0 and dir.type == 0xA500:
|
||||
aMemLength = mMemLength
|
||||
mMemLength = 0
|
||||
# hacky way to detect rels.arc
|
||||
|
||||
if folderCount == 2:
|
||||
fileCount = fileCount - 2 # need to check on the logic for this
|
||||
|
||||
arcHeader = RARC(
|
||||
1380012611,
|
||||
fileLength,
|
||||
32,
|
||||
fileOffset,
|
||||
len(data),
|
||||
mMemLength,
|
||||
aMemLength,
|
||||
0,
|
||||
len(nodes),
|
||||
32,
|
||||
len(dirs),
|
||||
dirOffset,
|
||||
stringTableLen + stringTablePadding,
|
||||
stringTableOffset,
|
||||
fileCount,
|
||||
256,
|
||||
0,
|
||||
)
|
||||
headerData = struct.pack(
|
||||
">IIIIIIIIIIIIIIHHI",
|
||||
1380012611,
|
||||
fileLength,
|
||||
32,
|
||||
fileOffset,
|
||||
len(data),
|
||||
mMemLength,
|
||||
aMemLength,
|
||||
0,
|
||||
len(nodes),
|
||||
32,
|
||||
len(dirs),
|
||||
dirOffset,
|
||||
stringTableLen + fileOffsetPadding,
|
||||
stringTableOffset,
|
||||
fileCount,
|
||||
256,
|
||||
0,
|
||||
)
|
||||
nodeData = bytearray()
|
||||
for node in nodes:
|
||||
nodeData = nodeData + struct.pack(
|
||||
">IIHHI",
|
||||
node.identifier,
|
||||
node.name_offset,
|
||||
node.name_hash,
|
||||
node.directory_count,
|
||||
node.directory_index,
|
||||
)
|
||||
|
||||
dirOffsetPaddingData = bytearray(dirOffsetPadding)
|
||||
dirData = bytearray()
|
||||
for dir in dirs:
|
||||
dirData = dirData + struct.pack(
|
||||
">HHHHIII",
|
||||
dir.index,
|
||||
dir.name_hash,
|
||||
dir.type,
|
||||
dir.name_offset,
|
||||
dir.data_offset,
|
||||
dir.data_length,
|
||||
dir.unknown0,
|
||||
)
|
||||
|
||||
stringTablePaddingData = bytearray(stringTablePadding)
|
||||
stringTableData = bytearray(stringTable, "shift-jis")
|
||||
fileOffsetPaddingData = bytearray(fileOffsetPadding)
|
||||
|
||||
fullData = bytearray()
|
||||
fullData = (
|
||||
headerData
|
||||
+ nodeData
|
||||
+ dirOffsetPaddingData
|
||||
+ dirData
|
||||
+ stringTablePaddingData
|
||||
+ stringTableData
|
||||
+ fileOffsetPaddingData
|
||||
+ data
|
||||
)
|
||||
|
||||
return fullData
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import struct
|
||||
import os
|
||||
import yaz0
|
||||
import libyaz0
|
||||
import io
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
@@ -149,7 +149,7 @@ def read_section(index, buffer):
|
||||
def read(buffer):
|
||||
# check if the rel is compressed
|
||||
if struct.unpack('>I', buffer[:4])[0] == 0x59617A30:
|
||||
buffer = yaz0.decompress(io.BytesIO(buffer))
|
||||
buffer = libyaz0.decompress(io.BytesIO(buffer))
|
||||
|
||||
header_size = 0x40
|
||||
header = struct.unpack('>IIIIIIIIIIIIBBBBIII', buffer[:0x40])
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
from .yaz0 import *
|
||||
@@ -0,0 +1,33 @@
|
||||
import ctypes
|
||||
import struct
|
||||
|
||||
_yaz0lib = ctypes.cdll.LoadLibrary("build/yaz0.so")
|
||||
|
||||
if _yaz0lib == None:
|
||||
print("Error: build/yaz0.so failed to load!")
|
||||
|
||||
def decompress(data):
|
||||
header = data[0:4]
|
||||
if header != bytearray("Yaz0","ascii"):
|
||||
return None
|
||||
decompressedSize = struct.unpack(">I",data[4:8])[0]
|
||||
compressedDataBuffer = ctypes.c_buffer(bytes(data[16:]))
|
||||
decompressedDataBuffer = ctypes.c_buffer(bytes(decompressedSize))
|
||||
decode = _yaz0lib.yaz0_decode
|
||||
decode.argtypes = [ctypes.c_char_p,ctypes.c_char_p,ctypes.c_int]
|
||||
decode.restype = ctypes.c_int
|
||||
decode(compressedDataBuffer,decompressedDataBuffer,decompressedSize)
|
||||
return bytearray(decompressedDataBuffer)[:-1]
|
||||
|
||||
def compress(data):
|
||||
decompresseddDataBuffer = ctypes.c_buffer(data)
|
||||
compressedDataBuffer = ctypes.c_buffer(bytes(len(data)*2))
|
||||
encode = _yaz0lib.yaz0_encode
|
||||
encode.argtypes = [ctypes.c_char_p,ctypes.c_char_p,ctypes.c_int]
|
||||
encode.restype = ctypes.c_int
|
||||
size = encode(decompresseddDataBuffer,compressedDataBuffer,len(data))
|
||||
|
||||
header_padding = bytearray(8)
|
||||
ident = bytearray("Yaz0","ascii")
|
||||
sizeInt = struct.pack(">I",len(data))
|
||||
return ident + sizeInt + header_padding + bytearray(compressedDataBuffer)[:size]
|
||||
+142
-324
@@ -3,28 +3,96 @@ import sys
|
||||
import shutil
|
||||
import extract_game_assets
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
import struct
|
||||
import ctypes
|
||||
import oead
|
||||
import libyaz0
|
||||
import libarc
|
||||
import threading
|
||||
|
||||
|
||||
def copy(path, destPath):
|
||||
def getMaxDateFromDir(path):
|
||||
maxTime = 0
|
||||
for root, dirs, files in os.walk(str(path)):
|
||||
for file in files:
|
||||
outputDir = destPath / Path(str(root))
|
||||
# print(str(outputDir.absolute())+file)
|
||||
if not outputDir.absolute().exists():
|
||||
os.makedirs(outputDir.absolute())
|
||||
outputFile = Path(str(outputDir.absolute()) + "/" + str(file))
|
||||
inFile = Path(str(Path(root).absolute()) + "/" + str(file))
|
||||
if not outputFile.exists():
|
||||
print(str(inFile) + " -> " + str(outputFile))
|
||||
shutil.copyfile(inFile, outputFile)
|
||||
else:
|
||||
if os.path.getmtime(inFile) > os.path.getmtime(outputFile):
|
||||
print(str(inFile) + " -> " + str(outputFile))
|
||||
shutil.copyfile(inFile, outputFile)
|
||||
time = os.path.getmtime(Path(root + "/" + file))
|
||||
if time > maxTime:
|
||||
maxTime = time
|
||||
return maxTime
|
||||
|
||||
|
||||
convertDefinitions = [
|
||||
{
|
||||
"sourceExtension": ".arc",
|
||||
"destExtension": ".arc",
|
||||
"convertFunction": libarc.convert_dir_to_arc,
|
||||
"exceptions": ["game/files/res/Object/HomeBtn.c.arc/archive/dat/speakerse.arc"],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def convertEntry(file, path, destPath, returnData):
|
||||
split = os.path.splitext(file)
|
||||
mustBeCompressed = False
|
||||
destFileName = file
|
||||
if split[0].split(".")[-1] == "c":
|
||||
destFileName = split[0][0:-2] + split[-1]
|
||||
mustBeCompressed = True
|
||||
sourceExtension = split[-1]
|
||||
data = None
|
||||
|
||||
extractDef = None
|
||||
for extractData in convertDefinitions:
|
||||
if sourceExtension == extractData["sourceExtension"]:
|
||||
extractDef = extractData
|
||||
if extractData["exceptions"] != None:
|
||||
for exception in extractData["exceptions"]:
|
||||
if str(path / file) == exception:
|
||||
extractDef = None
|
||||
break
|
||||
|
||||
if extractDef != None:
|
||||
destFileName = os.path.splitext(destFileName)[0] + extractDef["destExtension"]
|
||||
|
||||
targetTime = 0
|
||||
if destPath != None and os.path.exists(destPath / destFileName):
|
||||
targetTime = os.path.getmtime(destPath / destFileName)
|
||||
sourceTime = 0
|
||||
if targetTime != 0:
|
||||
if os.path.isdir(path / file):
|
||||
sourceTime = getMaxDateFromDir(path / file)
|
||||
else:
|
||||
sourceTime = os.path.getmtime(path / file)
|
||||
if returnData == False and sourceTime < targetTime:
|
||||
return destFileName
|
||||
|
||||
if extractDef != None:
|
||||
data = extractDef["convertFunction"](path / file, convertEntry)
|
||||
|
||||
if mustBeCompressed == True:
|
||||
if data == None:
|
||||
data = open(path / file, "rb").read()
|
||||
data = libyaz0.compress(data)
|
||||
if returnData == True:
|
||||
if data == None and returnData == True:
|
||||
data = open(path / file, "rb").read()
|
||||
return destFileName, data
|
||||
else:
|
||||
print(str(path / file) + " -> " + str(destPath / destFileName))
|
||||
if data != None:
|
||||
open(destPath / destFileName, "wb").write(data)
|
||||
else:
|
||||
shutil.copy(path / file, destPath / destFileName)
|
||||
return destFileName
|
||||
|
||||
def copy(path, destPath):
|
||||
for file in os.listdir(path):
|
||||
split = os.path.splitext(file)
|
||||
if len(split[1]) == 0 and os.path.isdir(path / file):
|
||||
# is a standard directory
|
||||
if not Path(destPath / file).exists():
|
||||
os.mkdir(destPath / file)
|
||||
copy(path / file, destPath / file)
|
||||
else:
|
||||
# either a file or directory that needs to be converted
|
||||
convertEntry(file, path, destPath, False)
|
||||
|
||||
|
||||
aMemRels = """d_a_alldie.rel
|
||||
@@ -163,89 +231,10 @@ d_a_vrbox.rel
|
||||
d_a_vrbox2.rel
|
||||
f_pc_profile_lst.rel"""
|
||||
|
||||
# Because libarc is only geared toward reading from arcs I'm writing my own arc writer in this file
|
||||
|
||||
|
||||
class HEADER:
|
||||
RARC: int
|
||||
length: int
|
||||
headerLength: int
|
||||
fileDataOffset: int
|
||||
fileDataLen: int
|
||||
fileDataLen2: int
|
||||
unk1: int
|
||||
unk2: int
|
||||
|
||||
|
||||
class INFO:
|
||||
numNodes: int
|
||||
firstNodeOffset: int
|
||||
totalDirNum: int
|
||||
firstDirOffset: int
|
||||
stringTableLen: int
|
||||
stringTableOffset: int
|
||||
numDirsThatAreFiles: int
|
||||
unk1: int
|
||||
unk2: int
|
||||
|
||||
|
||||
class NODE:
|
||||
NAME: int
|
||||
stringTableOffset: int
|
||||
hash: int
|
||||
numDirs: int
|
||||
firstDirIndex: int
|
||||
|
||||
|
||||
class DIRECTORY:
|
||||
dirIndex: int
|
||||
stringHash: int
|
||||
type: int
|
||||
stringOffset: int
|
||||
fileOffset: int
|
||||
fileLength: int
|
||||
unk1: int
|
||||
|
||||
|
||||
def computeHash(string):
|
||||
hash = 0
|
||||
for char in string:
|
||||
hash = hash * 3
|
||||
hash = hash + ord(char)
|
||||
|
||||
hash = ctypes.c_ushort(hash)
|
||||
hash = hash.value
|
||||
return hash
|
||||
|
||||
|
||||
def addFile(index, sizeIndex, dirs, name, stringTable, paths, data):
|
||||
file = DIRECTORY()
|
||||
file.dirIndex = index
|
||||
file.stringHash = computeHash(name)
|
||||
file.type = 0xA500
|
||||
file.stringOffset = stringTable.find(name)
|
||||
path = None
|
||||
for relPath in paths:
|
||||
if str(relPath).find(name) != -1:
|
||||
path = relPath
|
||||
file.unk1 = 0
|
||||
fileData = open(path, "rb")
|
||||
compressedData = oead.yaz0.compress(fileData.read())
|
||||
padding = 0x20 - (len(compressedData) % 0x20)
|
||||
file.fileLength = len(compressedData)
|
||||
file.fileOffset = sizeIndex
|
||||
sizeIndex = sizeIndex + file.fileLength + padding
|
||||
data += compressedData
|
||||
data += bytearray(padding)
|
||||
fileData.close()
|
||||
dirs.append(file)
|
||||
|
||||
return dirs, data, sizeIndex
|
||||
|
||||
|
||||
def copyRelFiles(buildPath, aMemList, mMemList):
|
||||
def copyRelFiles(gamePath, buildPath, aMemList, mMemList):
|
||||
relArcPaths = []
|
||||
for root, dirs, files in os.walk(str(buildPath / "rel")):
|
||||
for root, dirs, files in os.walk(str(buildPath / "dolzel2/rel")):
|
||||
for file in files:
|
||||
if file.find(".rel") != -1:
|
||||
relArcFound = False
|
||||
@@ -260,14 +249,14 @@ def copyRelFiles(buildPath, aMemList, mMemList):
|
||||
print(
|
||||
str(fullPath)
|
||||
+ " -> "
|
||||
+ str(buildPath / "game/files/rel/Final/Release" / file)
|
||||
+ str(buildPath / "dolzel2/game/files/rel/Final/Release" / file)
|
||||
)
|
||||
relSource = open(fullPath, "rb")
|
||||
data = relSource.read()
|
||||
relSource.close()
|
||||
data = oead.yaz0.compress(data)
|
||||
data = libyaz0.compress(data)
|
||||
relNew = open(
|
||||
buildPath / "game/files/rel/Final/Release" / file, "wb"
|
||||
buildPath / "dolzel2/game/files/rel/Final/Release" / file, "wb"
|
||||
)
|
||||
relNew.write(data)
|
||||
relNew.truncate()
|
||||
@@ -275,225 +264,45 @@ def copyRelFiles(buildPath, aMemList, mMemList):
|
||||
else:
|
||||
relArcPaths.append(fullPath)
|
||||
|
||||
arcHeader = HEADER()
|
||||
arcHeader.RARC = 0x52415243
|
||||
arcHeader.headerLength = 0x20
|
||||
arcHeader.unk1 = 0
|
||||
arcHeader.unk2 = 0
|
||||
infoBlock = INFO()
|
||||
infoBlock.numNodes = 3
|
||||
infoBlock.numDirsThatAreFiles = 142
|
||||
rootNode = NODE()
|
||||
rootNode.NAME = 0x524F4F54
|
||||
rootNode.numDirs = 4
|
||||
rootNode.firstDirIndex = 0
|
||||
rootNode.hash = computeHash("rels")
|
||||
aMemNode = NODE()
|
||||
aMemNode.NAME = 0x414D454D
|
||||
aMemNode.hash = computeHash("amem")
|
||||
aMemNode.numDirs = 79
|
||||
aMemNode.firstDirIndex = 4
|
||||
mMemNode = NODE()
|
||||
mMemNode.hash = computeHash("mmem")
|
||||
mMemNode.NAME = 0x4D4D454D
|
||||
mMemNode.numDirs = 59
|
||||
mMemNode.firstDirIndex = 83
|
||||
if os.path.exists(buildPath / "RELS.arc") == False:
|
||||
os.mkdir(buildPath / "RELS.arc")
|
||||
os.mkdir(buildPath / "RELS.arc/rels")
|
||||
os.mkdir(buildPath / "RELS.arc/rels/mmem")
|
||||
os.mkdir(buildPath / "RELS.arc/rels/amem")
|
||||
|
||||
stringTable = ".\0..\0rels\0amem\0"
|
||||
for rel in aMemList:
|
||||
stringTable = stringTable + rel + "\0"
|
||||
stringTable = stringTable + "mmem\0"
|
||||
for rel in mMemList:
|
||||
stringTable = stringTable + rel + "\0"
|
||||
stringTable = stringTable + "\0\0\0\0\0\0"
|
||||
|
||||
rootNode.stringTableOffset = stringTable.find("rels")
|
||||
aMemNode.stringTableOffset = stringTable.find("amem")
|
||||
mMemNode.stringTableOffset = stringTable.find("mmem")
|
||||
|
||||
aMemDir = DIRECTORY()
|
||||
aMemDir.dirIndex = 0xFFFF
|
||||
aMemDir.type = 0x200
|
||||
aMemDir.stringOffset = stringTable.find("amem")
|
||||
aMemDir.stringHash = computeHash("amem")
|
||||
aMemDir.fileOffset = 1
|
||||
aMemDir.fileLength = 0x10
|
||||
aMemDir.unk1 = 0
|
||||
|
||||
mMemDir = DIRECTORY()
|
||||
mMemDir.dirIndex = 0xFFFF
|
||||
mMemDir.type = 0x200
|
||||
mMemDir.stringOffset = stringTable.find("mmem")
|
||||
mMemDir.stringHash = computeHash("mmem")
|
||||
mMemDir.fileOffset = 2
|
||||
mMemDir.fileLength = 0x10
|
||||
mMemDir.unk1 = 0
|
||||
|
||||
unkDir = DIRECTORY()
|
||||
unkDir.dirIndex = 0xFFFF
|
||||
unkDir.stringHash = 0x2E
|
||||
unkDir.type = 0x200
|
||||
unkDir.stringOffset = 0
|
||||
unkDir.fileOffset = 0
|
||||
unkDir.fileLength = 0x10
|
||||
unkDir.unk1 = 0
|
||||
|
||||
unkDir2 = DIRECTORY()
|
||||
unkDir2.dirIndex = 0xFFFF
|
||||
unkDir2.stringHash = 0xB8
|
||||
unkDir2.type = 0x200
|
||||
unkDir2.stringOffset = 2
|
||||
unkDir2.fileOffset = 0xFFFFFFFF
|
||||
unkDir2.fileLength = 0x10
|
||||
unkDir2.unk1 = 0
|
||||
|
||||
dirs = [aMemDir, mMemDir, unkDir, unkDir2]
|
||||
|
||||
data = bytearray()
|
||||
|
||||
dirIndex = 4
|
||||
sizeIndex = 0
|
||||
for rel in aMemList:
|
||||
retdirs, retdata, retSize = addFile(
|
||||
dirIndex, sizeIndex, dirs, rel, stringTable, relArcPaths, data
|
||||
)
|
||||
dirIndex = dirIndex + 1
|
||||
sizeIndex = retSize
|
||||
dirs = retdirs
|
||||
data = retdata
|
||||
dirs.append(unkDir)
|
||||
dirs.append(unkDir2)
|
||||
dirIndex = dirIndex + 2
|
||||
for rel in mMemList:
|
||||
retdirs, retdata, retSize = addFile(
|
||||
dirIndex, sizeIndex, dirs, rel, stringTable, relArcPaths, data
|
||||
)
|
||||
dirIndex = dirIndex + 1
|
||||
sizeIndex = retSize
|
||||
# print(hex(dirIndex))
|
||||
dirs = retdirs
|
||||
data = retdata
|
||||
unkDir3 = DIRECTORY()
|
||||
unkDir3.dirIndex = 0xFFFF
|
||||
unkDir3.stringHash = 0x2E
|
||||
unkDir3.type = 0x200
|
||||
unkDir3.stringOffset = 0
|
||||
unkDir3.fileOffset = 2
|
||||
unkDir3.fileLength = 0x10
|
||||
unkDir3.unk1 = 0
|
||||
|
||||
unkDir4 = DIRECTORY()
|
||||
unkDir4.dirIndex = 0xFFFF
|
||||
unkDir4.stringHash = 0xB8
|
||||
unkDir4.type = 0x200
|
||||
unkDir4.stringOffset = 2
|
||||
unkDir4.fileOffset = 0
|
||||
unkDir4.fileLength = 0x10
|
||||
unkDir4.unk1 = 0
|
||||
dirs.append(unkDir3)
|
||||
dirs.append(unkDir4)
|
||||
dirIndex = dirIndex + 2
|
||||
|
||||
arcHeader.length = (
|
||||
len(stringTable) + 0x20 + 0x20 + 0x30 + (len(dirs) * 0x14) + len(data)
|
||||
filesTxtData = (
|
||||
"Folder:rels/amem/\nFolder:rels/mmem/\nFolder:rels/./\nFolder:rels/../\n"
|
||||
)
|
||||
arcHeader.fileDataOffset = 0x14E0
|
||||
arcHeader.fileDataLen = len(data)
|
||||
arcHeader.fileDataLen2 = arcHeader.fileDataLen
|
||||
for i, rel in enumerate(aMemRels.splitlines()):
|
||||
filesTxtData = filesTxtData + str(i + 4) + ":rels/amem/" + rel + ":0xa500\n"
|
||||
filesTxtData = filesTxtData + "Folder:rels/amem/./\nFolder:rels/amem/../\n"
|
||||
for i, rel in enumerate(mMemRels.splitlines()):
|
||||
filesTxtData = filesTxtData + str(i + 83) + ":rels/mmem/" + rel + ":0xa500\n"
|
||||
filesTxtData = filesTxtData + "Folder:rels/mmem/./\nFolder:rels/mmem/../\n"
|
||||
open(buildPath / "RELS.arc/_files.txt", "w").write(filesTxtData)
|
||||
for rel in relArcPaths:
|
||||
for rel2 in aMemRels.splitlines():
|
||||
if str(rel).find(rel2) != -1:
|
||||
sourceRel = open(rel, "rb").read()
|
||||
open(buildPath / "RELS.arc/rels/amem/" / rel2, "wb").write(
|
||||
libyaz0.compress(sourceRel)
|
||||
)
|
||||
break
|
||||
for rel2 in mMemRels.splitlines():
|
||||
if str(rel).find(rel2) != -1:
|
||||
sourceRel = open(rel, "rb").read()
|
||||
open(buildPath / "RELS.arc/rels/mmem/" / rel2, "wb").write(
|
||||
libyaz0.compress(sourceRel)
|
||||
)
|
||||
break
|
||||
|
||||
infoBlock.firstNodeOffset = 0x20
|
||||
infoBlock.firstDirOffset = 0x60
|
||||
infoBlock.stringTableLen = len(stringTable)
|
||||
infoBlock.stringTableOffset = 0xB80
|
||||
infoBlock.unk1 = 0x100
|
||||
infoBlock.unk2 = 0
|
||||
infoBlock.totalDirNum = 0x8E
|
||||
|
||||
outputArcFile = open(buildPath / "game/files/RELS.arc", "wb")
|
||||
outputArcFile.seek(0)
|
||||
|
||||
outputArcFile.write(
|
||||
struct.pack(
|
||||
">IIIIIIII",
|
||||
arcHeader.RARC,
|
||||
arcHeader.length,
|
||||
arcHeader.headerLength,
|
||||
arcHeader.fileDataOffset,
|
||||
arcHeader.fileDataLen,
|
||||
arcHeader.unk1,
|
||||
arcHeader.fileDataLen2,
|
||||
arcHeader.unk2,
|
||||
)
|
||||
print("Creating RELS.arc")
|
||||
open(buildPath / "dolzel2/game/files/RELS.arc", "wb").write(
|
||||
libarc.convert_dir_to_arc(buildPath / "RELS.arc", convertEntry)
|
||||
)
|
||||
outputArcFile.write(
|
||||
struct.pack(
|
||||
">IIIIIIHHI",
|
||||
infoBlock.numNodes,
|
||||
infoBlock.firstNodeOffset,
|
||||
infoBlock.totalDirNum,
|
||||
infoBlock.firstDirOffset,
|
||||
infoBlock.stringTableLen,
|
||||
infoBlock.stringTableOffset,
|
||||
infoBlock.numDirsThatAreFiles,
|
||||
infoBlock.unk1,
|
||||
infoBlock.unk2,
|
||||
)
|
||||
)
|
||||
outputArcFile.write(
|
||||
struct.pack(
|
||||
">IIHHI",
|
||||
rootNode.NAME,
|
||||
rootNode.stringTableOffset,
|
||||
rootNode.hash,
|
||||
rootNode.numDirs,
|
||||
rootNode.firstDirIndex,
|
||||
)
|
||||
)
|
||||
outputArcFile.write(
|
||||
struct.pack(
|
||||
">IIHHI",
|
||||
aMemNode.NAME,
|
||||
aMemNode.stringTableOffset,
|
||||
aMemNode.hash,
|
||||
aMemNode.numDirs,
|
||||
aMemNode.firstDirIndex,
|
||||
)
|
||||
)
|
||||
outputArcFile.write(
|
||||
struct.pack(
|
||||
">IIHHI",
|
||||
mMemNode.NAME,
|
||||
mMemNode.stringTableOffset,
|
||||
mMemNode.hash,
|
||||
mMemNode.numDirs,
|
||||
mMemNode.firstDirIndex,
|
||||
)
|
||||
)
|
||||
outputArcFile.write(bytearray(16))
|
||||
for dir in dirs:
|
||||
outputArcFile.write(
|
||||
struct.pack(
|
||||
">HHHHIII",
|
||||
dir.dirIndex,
|
||||
dir.stringHash,
|
||||
dir.type,
|
||||
dir.stringOffset,
|
||||
dir.fileOffset,
|
||||
dir.fileLength,
|
||||
dir.unk1,
|
||||
)
|
||||
)
|
||||
outputArcFile.write(bytearray(8))
|
||||
strBytearray = bytearray()
|
||||
strBytearray.extend(map(ord, stringTable))
|
||||
outputArcFile.write(strBytearray)
|
||||
outputArcFile.write(data)
|
||||
|
||||
outputArcFile.truncate()
|
||||
outputArcFile.close()
|
||||
|
||||
|
||||
def main(gamePath, buildPath):
|
||||
def main(gamePath, buildPath, copyCode):
|
||||
if not gamePath.exists():
|
||||
gamePath.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -510,17 +319,26 @@ def main(gamePath, buildPath):
|
||||
os.chdir(previousDir)
|
||||
|
||||
print("Copying game files...")
|
||||
copy(gamePath, buildPath.absolute())
|
||||
if os.path.exists(buildPath / "dolzel2") == False:
|
||||
os.mkdir(buildPath / "dolzel2")
|
||||
if os.path.exists(buildPath / "dolzel2" / "game") == False:
|
||||
os.mkdir(buildPath / "dolzel2/game")
|
||||
copy(gamePath, Path(buildPath / "dolzel2/game").absolute())
|
||||
|
||||
print(
|
||||
str(buildPath / "main_shift.dol")
|
||||
+ " -> "
|
||||
+ str(buildPath / "game/sys/main.dol")
|
||||
)
|
||||
shutil.copyfile(buildPath / "main_shift.dol", buildPath / "game/sys/main.dol")
|
||||
if copyCode != "noCopyCode":
|
||||
print(
|
||||
str(buildPath / "dolzel2/main_shift.dol")
|
||||
+ " -> "
|
||||
+ str(buildPath / "dolzel2/game/sys/main.dol")
|
||||
)
|
||||
shutil.copyfile(
|
||||
buildPath / "dolzel2/main_shift.dol",
|
||||
buildPath / "dolzel2/game/sys/main.dol",
|
||||
)
|
||||
|
||||
copyRelFiles(buildPath, aMemRels.splitlines(), mMemRels.splitlines())
|
||||
copyRelFiles(gamePath, buildPath, aMemRels.splitlines(), mMemRels.splitlines())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(Path(sys.argv[1]), Path(sys.argv[2]))
|
||||
pass
|
||||
main(Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3])
|
||||
|
||||
@@ -10,4 +10,4 @@ cxxfilt
|
||||
pyelftools
|
||||
requests
|
||||
zipfile
|
||||
shutil
|
||||
shutil
|
||||
@@ -0,0 +1,7 @@
|
||||
YAZ0_CC := cc
|
||||
YAZ0_CFLAGS := -fPIC -shared -O3 -Wall -s
|
||||
|
||||
$(YAZ0): include tools/yaz0/Makefile
|
||||
@echo [tools] building yaz0.so
|
||||
@$(YAZ0_CC) $(YAZ0_CFLAGS) -o $(YAZ0) tools/yaz0/yaz0.c
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
#include <assert.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
//yaz0 implementation by BluRose
|
||||
//patched by Prakxo
|
||||
|
||||
// decoder implementation by thakis of http://www.amnoid.de
|
||||
|
||||
// src points to the yaz0 source data (to the "real" source data, not at the header!)
|
||||
// dst points to a buffer uncompressedSize bytes large (you get uncompressedSize from
|
||||
// the second 4 bytes in the Yaz0 header).
|
||||
int yaz0_decode(uint8_t* src, uint8_t* dst, int uncompressedSize)
|
||||
{
|
||||
int srcPlace = 0, dstPlace = 0; // current read/write positions
|
||||
|
||||
unsigned int validBitCount = 0; // number of valid bits left in "code" byte
|
||||
uint8_t currCodeByte;
|
||||
while (dstPlace < uncompressedSize)
|
||||
{
|
||||
// read new "code" byte if the current one is used up
|
||||
if (validBitCount == 0)
|
||||
{
|
||||
currCodeByte = src[srcPlace];
|
||||
++srcPlace;
|
||||
validBitCount = 8;
|
||||
}
|
||||
|
||||
if ((currCodeByte & 0x80) != 0)
|
||||
{
|
||||
// straight copy
|
||||
dst[dstPlace] = src[srcPlace];
|
||||
dstPlace++;
|
||||
srcPlace++;
|
||||
}
|
||||
else
|
||||
{
|
||||
// RLE part
|
||||
uint8_t byte1 = src[srcPlace];
|
||||
uint8_t byte2 = src[srcPlace + 1];
|
||||
srcPlace += 2;
|
||||
|
||||
unsigned int dist = ((byte1 & 0xF) << 8) | byte2;
|
||||
unsigned int copySource = dstPlace - (dist + 1);
|
||||
|
||||
unsigned int numBytes = byte1 >> 4;
|
||||
if (numBytes == 0)
|
||||
{
|
||||
numBytes = src[srcPlace] + 0x12;
|
||||
srcPlace++;
|
||||
}
|
||||
else
|
||||
{
|
||||
numBytes += 2;
|
||||
}
|
||||
|
||||
// copy run
|
||||
for (unsigned int i = 0; i < numBytes; ++i)
|
||||
{
|
||||
dst[dstPlace] = dst[copySource];
|
||||
copySource++;
|
||||
dstPlace++;
|
||||
}
|
||||
}
|
||||
|
||||
// use next bit from "code" byte
|
||||
currCodeByte <<= 1;
|
||||
validBitCount -= 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// encoder implementation by shevious, with bug fixes by notwa
|
||||
|
||||
typedef uint32_t uint32_t;
|
||||
typedef uint8_t uint8_t;
|
||||
|
||||
#define MAX_RUNLEN (0xFF + 0x12)
|
||||
|
||||
// simple and straight encoding scheme for Yaz0
|
||||
static uint32_t simpleEnc(uint8_t *src, int size, int pos, uint32_t *pMatchPos)
|
||||
{
|
||||
int numBytes = 1;
|
||||
int matchPos = 0;
|
||||
|
||||
int startPos = pos - 0x1000;
|
||||
int end = size - pos;
|
||||
|
||||
if (startPos < 0)
|
||||
startPos = 0;
|
||||
|
||||
// maximum runlength for 3 byte encoding
|
||||
if (end > MAX_RUNLEN)
|
||||
end = MAX_RUNLEN;
|
||||
|
||||
for (int i = startPos; i < pos; i++)
|
||||
{
|
||||
int j;
|
||||
|
||||
for (j = 0; j < end; j++)
|
||||
{
|
||||
if (src[i + j] != src[j + pos])
|
||||
break;
|
||||
}
|
||||
if (j > numBytes)
|
||||
{
|
||||
numBytes = j;
|
||||
matchPos = i;
|
||||
}
|
||||
}
|
||||
|
||||
*pMatchPos = matchPos;
|
||||
|
||||
if (numBytes == 2)
|
||||
numBytes = 1;
|
||||
|
||||
return numBytes;
|
||||
}
|
||||
|
||||
// a lookahead encoding scheme for ngc Yaz0
|
||||
static uint32_t nintendoEnc(uint8_t *src, int size, int pos, uint32_t *pMatchPos)
|
||||
{
|
||||
uint32_t numBytes = 1;
|
||||
static uint32_t numBytes1;
|
||||
static uint32_t matchPos;
|
||||
static int prevFlag = 0;
|
||||
|
||||
// if prevFlag is set, it means that the previous position
|
||||
// was determined by look-ahead try.
|
||||
// so just use it. this is not the best optimization,
|
||||
// but nintendo's choice for speed.
|
||||
if (prevFlag == 1)
|
||||
{
|
||||
*pMatchPos = matchPos;
|
||||
prevFlag = 0;
|
||||
return numBytes1;
|
||||
}
|
||||
|
||||
prevFlag = 0;
|
||||
numBytes = simpleEnc(src, size, pos, &matchPos);
|
||||
*pMatchPos = matchPos;
|
||||
|
||||
// if this position is RLE encoded, then compare to copying 1 byte and next position(pos+1) encoding
|
||||
if (numBytes >= 3)
|
||||
{
|
||||
numBytes1 = simpleEnc(src, size, pos + 1, &matchPos);
|
||||
// if the next position encoding is +2 longer than current position, choose it.
|
||||
// this does not guarantee the best optimization, but fairly good optimization with speed.
|
||||
if (numBytes1 >= numBytes + 2)
|
||||
{
|
||||
numBytes = 1;
|
||||
prevFlag = 1;
|
||||
}
|
||||
}
|
||||
return numBytes;
|
||||
}
|
||||
|
||||
int yaz0_encode(uint8_t *src, uint8_t *dst, int srcSize)
|
||||
{
|
||||
int srcPos = 0;
|
||||
int dstPos = 0;
|
||||
int bufPos = 0;
|
||||
|
||||
uint8_t buf[24]; // 8 codes * 3 bytes maximum
|
||||
|
||||
uint32_t validBitCount = 0; // number of valid bits left in "code" byte
|
||||
uint8_t currCodeByte = 0; // a bitfield, set bits meaning copy, unset meaning RLE
|
||||
|
||||
while (srcPos < srcSize)
|
||||
{
|
||||
uint32_t numBytes;
|
||||
uint32_t matchPos;
|
||||
|
||||
numBytes = nintendoEnc(src, srcSize, srcPos, &matchPos);
|
||||
if (numBytes < 3)
|
||||
{
|
||||
// straight copy
|
||||
buf[bufPos] = src[srcPos];
|
||||
bufPos++;
|
||||
srcPos++;
|
||||
//set flag for straight copy
|
||||
currCodeByte |= (0x80 >> validBitCount);
|
||||
}
|
||||
else
|
||||
{
|
||||
//RLE part
|
||||
uint32_t dist = srcPos - matchPos - 1;
|
||||
uint8_t byte1, byte2, byte3;
|
||||
|
||||
if (numBytes >= 0x12) // 3 byte encoding
|
||||
{
|
||||
byte1 = 0 | (dist >> 8);
|
||||
byte2 = dist & 0xFF;
|
||||
buf[bufPos++] = byte1;
|
||||
buf[bufPos++] = byte2;
|
||||
// maximum runlength for 3 byte encoding
|
||||
if (numBytes > MAX_RUNLEN)
|
||||
numBytes = MAX_RUNLEN;
|
||||
byte3 = numBytes - 0x12;
|
||||
buf[bufPos++] = byte3;
|
||||
}
|
||||
else // 2 byte encoding
|
||||
{
|
||||
byte1 = ((numBytes - 2) << 4) | (dist >> 8);
|
||||
byte2 = dist & 0xFF;
|
||||
buf[bufPos++] = byte1;
|
||||
buf[bufPos++] = byte2;
|
||||
}
|
||||
srcPos += numBytes;
|
||||
}
|
||||
|
||||
validBitCount++;
|
||||
|
||||
// write eight codes
|
||||
if (validBitCount == 8)
|
||||
{
|
||||
dst[dstPos++] = currCodeByte;
|
||||
for (int j = 0; j < bufPos; j++)
|
||||
dst[dstPos++] = buf[j];
|
||||
|
||||
currCodeByte = 0;
|
||||
validBitCount = 0;
|
||||
bufPos = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (validBitCount > 0)
|
||||
{
|
||||
dst[dstPos++] = currCodeByte;
|
||||
for (int j = 0; j < bufPos; j++)
|
||||
dst[dstPos++] = buf[j];
|
||||
|
||||
currCodeByte = 0;
|
||||
validBitCount = 0;
|
||||
bufPos = 0;
|
||||
}
|
||||
|
||||
return dstPos;
|
||||
}
|
||||
Reference in New Issue
Block a user