Assets system: reconcile xml versions using implicit and relative offsets (#2614)

* Assets system: reconcile xml versions using implicit offsets

* simplify

* remove other object_mag xmls

* Introduce relative offsets: `Offset=".+0x421"`, more sizable `ResourceDesc`s, update spec

* fix

* reconcile object_mo xml

* reconcile object_tw xml

* reconcile object_xc xml (and add Tlut="gNameTLUT" to Texture resources)

* reconcile object_fd xml

* reconcile object_kingdodongo xml

* bss

* reconcile gameplay_keep.xml
This commit is contained in:
Dragorn421
2026-05-09 22:36:05 +02:00
committed by GitHub
parent 9b795e46de
commit 4b742bf3cf
40 changed files with 1745 additions and 4663 deletions
+23 -2
View File
@@ -1,6 +1,7 @@
# SPDX-FileCopyrightText: © 2025 ZeldaRET
# SPDX-License-Identifier: CC0-1.0
import argparse
from pprint import pprint as vanilla_pprint
try:
@@ -14,17 +15,37 @@ from . import base
def main():
vc = version_config.load_version_config("gc-eu-mq-dbg")
parser = argparse.ArgumentParser()
parser.add_argument("--version", "-v", default="gc-eu-mq-dbg")
parser.add_argument("--all", action="store_true")
parser.add_argument("-s", dest="single", default=None)
args = parser.parse_args()
if not args.all and args.single is None:
parser.error("Must specify --all or -s")
vc = version_config.load_version_config(args.version)
pools = base.get_resources_desc(vc)
try:
for pool in pools:
if not args.all and not any(
coll.backing_memory.name == args.single
for coll in pool.collections
if isinstance(coll.backing_memory, base.BaseromFileBackingMemory)
):
continue
if any(coll.out_path.name == "gameplay_keep" for coll in pool.collections):
vanilla_pprint(pool)
else:
pprint(pool)
input("Press enter for next pool")
for coll in pool.collections:
print(coll.out_path)
for res in coll.resources:
print(f"0x{res.offset:06X}", res.symbol_name)
if args.all:
input("Press enter for next pool")
except KeyboardInterrupt:
print()
+44 -2
View File
@@ -5,6 +5,7 @@ import abc
import dataclasses
from functools import cache
from pathlib import Path
import re
from typing import Callable, Optional
from xml.etree import ElementTree
@@ -27,6 +28,10 @@ class NoBackingMemory(BackingMemory):
pass
class ResourceHasNoSizeError(Exception):
pass
# eq=False so this uses id-based equality and hashing
# Subclasses must also be made to use id-based equality and hashing
@dataclasses.dataclass(eq=False)
@@ -43,6 +48,9 @@ class ResourceDesc(abc.ABC):
hack_modes: set[str] = dataclasses.field(init=False, default_factory=set)
def get_size(self) -> int:
raise ResourceHasNoSizeError()
class StartAddress(abc.ABC):
pass
@@ -197,6 +205,15 @@ def get_resources_desc(vc: version_config.VersionConfig):
return pools
def _get_version_resources(fileelem: ElementTree.Element, version: str):
for reselem in fileelem:
if reselem.tag == "Version":
if re.fullmatch(reselem.attrib["Pattern"], version):
yield from reselem
else:
yield reselem
def _get_resources_fileelem_to_resourcescollection_pass1(
vc: version_config.VersionConfig,
pool: list[AssetConfigPiece],
@@ -257,10 +274,31 @@ def _get_resources_fileelem_to_resourcescollection_pass1(
[],
)
needs_pass2_exceptions: list[ResourceHandlerNeedsPass2Exception] = []
for reselem in fileelem:
prev_resource_end_offset = 0
for reselem in _get_version_resources(fileelem, vc.version):
try:
symbol_name = reselem.attrib["Name"]
offset = int(reselem.attrib["Offset"], 16)
if "Offset" in reselem.attrib:
offset_str = reselem.attrib["Offset"]
if offset_str.startswith(".+"):
if prev_resource_end_offset is None:
raise Exception(
f"Resource {symbol_name} has a relative Offset"
" and previous resource has no known end offset"
)
rel_offset = int(offset_str.removeprefix(".+"), 16)
offset = prev_resource_end_offset + rel_offset
else:
offset = int(offset_str, 16)
else:
if prev_resource_end_offset is None:
raise Exception(
f"Resource {symbol_name} has no Offset"
" and previous resource has no known end offset"
)
offset = prev_resource_end_offset
res_handler = _get_resource_handler(reselem.tag)
try:
res = res_handler(symbol_name, offset, collection, reselem)
@@ -269,6 +307,10 @@ def _get_resources_fileelem_to_resourcescollection_pass1(
needs_pass2_exceptions.append(needs_pass2_exc)
assert isinstance(res, ResourceDesc)
resources.append(res)
try:
prev_resource_end_offset = res.offset + res.get_size()
except ResourceHasNoSizeError:
prev_resource_end_offset = None
except Exception as e:
raise Exception(
"Error with resource element:\n"
+62 -15
View File
@@ -3,6 +3,7 @@
import dataclasses
import enum
from typing import Optional
from xml.etree.ElementTree import Element
from ..n64 import G_IM_FMT, G_IM_SIZ
@@ -13,6 +14,7 @@ from .base import (
ResourcesDescCollectionsPool,
ResourceHandlerNeedsPass2Exception,
BaseromFileBackingMemory,
ResourceHasNoSizeError,
)
from . import xml_errors
@@ -24,18 +26,29 @@ class GfxMicroCode(enum.Enum):
@dataclasses.dataclass(eq=False)
class DListResourceDesc(ResourceDesc):
length: Optional[int]
ucode: GfxMicroCode
raw_pointers: set[int] = dataclasses.field(default_factory=set)
"""Pointers in the dlist that are fine to keep raw ("in hex") instead of using symbols"""
def get_size(self):
if self.length is None:
raise ResourceHasNoSizeError()
return self.length * 8
def handler_DList(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset"}, {"Ucode", "RawPointers"})
xml_errors.check_attrib(
reselem, {"Name"}, {"Offset", "Length", "Ucode", "RawPointers"}
)
length = None
if "Length" in reselem.attrib:
length = int(reselem.attrib["Length"])
if "Ucode" in reselem.attrib:
ucode = GfxMicroCode[reselem.attrib["Ucode"].upper()]
else:
ucode = GfxMicroCode.F3DEX2
res = DListResourceDesc(symbol_name, offset, collection, reselem, ucode)
res = DListResourceDesc(symbol_name, offset, collection, reselem, length, ucode)
raw_pointers_str = reselem.attrib.get("RawPointers")
if raw_pointers_str:
for rp_str in raw_pointers_str.split(","):
@@ -47,9 +60,12 @@ def handler_DList(symbol_name, offset, collection, reselem: Element):
class BlobResourceDesc(ResourceDesc):
size: int
def get_size(self) -> int:
return self.size
def handler_Blob(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset", "Size"})
xml_errors.check_attrib(reselem, {"Name", "Size"}, {"Offset"})
size = int(reselem.attrib["Size"], 16)
return BlobResourceDesc(symbol_name, offset, collection, reselem, size)
@@ -60,7 +76,7 @@ class MtxResourceDesc(ResourceDesc):
def handler_Mtx(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset"})
xml_errors.check_attrib(reselem, {"Name"}, {"Offset"})
return MtxResourceDesc(symbol_name, offset, collection, reselem)
@@ -78,9 +94,12 @@ class Vec3sArrayResourceDesc(ResourceDesc):
class VtxArrayResourceDesc(ResourceDesc):
count: int
def get_size(self):
return self.count * 0x10
def handler_Array(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset", "Count"})
xml_errors.check_attrib(reselem, {"Name", "Count"}, {"Offset"})
count = int(reselem.attrib["Count"])
assert len(reselem) == 1, "Expected exactly one child of Array node"
array_elem = reselem[0]
@@ -121,6 +140,9 @@ class TextureResourceDesc(ResourceDesc):
width: int
height: int
def get_size(self):
return self.width * self.height * self.format.siz.bpp // 8
@dataclasses.dataclass(eq=False)
class CITextureResourceDesc(TextureResourceDesc):
@@ -132,10 +154,12 @@ def handler_Texture(
):
xml_errors.check_attrib(
reselem,
{"Name", "Offset", "Format", "Width", "Height"},
{"Name", "Format", "Width", "Height"},
# TODO remove SplitTlut
{
"Offset",
"SplitTlut",
"Tlut",
"TlutOffset",
"ExternalTlut",
"ExternalTlutOffset",
@@ -156,15 +180,39 @@ def handler_Texture(
res.hack_modes.add("hackmode_split_tlut_false")
assert (
"TlutOffset" in reselem.attrib or "ExternalTlutOffset" in reselem.attrib
), f"CI texture {symbol_name} is missing a tlut offset"
"Tlut" in reselem.attrib
or "TlutOffset" in reselem.attrib
or "ExternalTlutOffset" in reselem.attrib
), f"CI texture {symbol_name} is missing tlut information"
if "TlutOffset" in reselem.attrib:
if "Tlut" in reselem.attrib:
xml_errors.check_attrib(
reselem,
{"Name", "Offset", "Format", "Width", "Height", "TlutOffset"},
{"Name", "Format", "Width", "Height", "Tlut"},
# TODO remove SplitTlut
{"SplitTlut", "HackMode"},
{"Offset", "SplitTlut", "HackMode"},
)
tlut_name = reselem.attrib["Tlut"]
def pass2_callback(pool: ResourcesDescCollectionsPool):
matching_tlut_resources = [
res for res in collection.resources if res.symbol_name == tlut_name
]
assert len(matching_tlut_resources) == 1, (
f"Found {len(matching_tlut_resources)} resources named "
f"{tlut_name} instead of exactly one"
)
assert isinstance(
matching_tlut_resources[0], TextureResourceDesc
), matching_tlut_resources[0]
res.tlut = matching_tlut_resources[0]
elif "TlutOffset" in reselem.attrib:
xml_errors.check_attrib(
reselem,
{"Name", "Format", "Width", "Height", "TlutOffset"},
# TODO remove SplitTlut
{"Offset", "SplitTlut", "HackMode"},
)
tlut_offset = int(reselem.attrib["TlutOffset"], 16)
@@ -186,7 +234,6 @@ def handler_Texture(
reselem,
{
"Name",
"Offset",
"Format",
"Width",
"Height",
@@ -194,7 +241,7 @@ def handler_Texture(
"ExternalTlutOffset",
},
# TODO remove SplitTlut
{"SplitTlut", "HackMode"},
{"Offset", "SplitTlut", "HackMode"},
)
external_tlut_file = reselem.attrib["ExternalTlut"]
external_tlut_offset = int(reselem.attrib["ExternalTlutOffset"], 16)
@@ -222,8 +269,8 @@ def handler_Texture(
else:
xml_errors.check_attrib(
reselem,
{"Name", "Offset", "Format", "Width", "Height"},
{"HackMode"},
{"Name", "Format", "Width", "Height"},
{"Offset", "HackMode"},
)
res = TextureResourceDesc(
symbol_name, offset, collection, reselem, format, width, height
+18 -8
View File
@@ -44,20 +44,27 @@ For example, `baseroms/gc-eu/config.yml` contains
```yml
assets:
- name: objects/gameplay_keep
xml_path: assets/xml/objects/gameplay_keep_pal.xml
xml_path: assets/xml/objects/gameplay_keep.xml
```
then `<ExternalFile OutPath="assets/objects/gameplay_keep/"/>` refers to that gameplay_keep entry, which uses the `gameplay_keep_pal.xml` xml file when extracting assets for version gc-eu.
then `<ExternalFile OutPath="assets/objects/gameplay_keep/"/>` refers to that gameplay_keep entry, which uses the `gameplay_keep.xml` xml file when extracting assets for version gc-eu.
# Resource elements
Resource elements describe resources. Resources are pieces of data corresponding to a symbol each.
Two attributes are required on all resource elements: `Name` and `Offset`.
One attribute is required on all resource elements: `Name`.
- `Name` is the name of the symbol associated to the resource.
- `Offset` is the location in bytes from the start of the file data.
Another attribute, optional, is common to all resource elements: `Offset`.
If `Offset` is set to a (hexadecimal) number such as `Offset="0x421"`, it specifies the location of the resource in bytes from the start of the file data.
If `Offset` is not set, the resource is assumed to start where the previous resource ended, or at 0 if the resource is the first in the `<File>`.
`Offset` can also be set to be a relative offset like `Offset=".+0x421"`, which makes the resource location be that many bytes after the end of the previous resource.
## `Blob`
@@ -74,12 +81,14 @@ Unstructured binary data.
## `DList`
```xml
<DList Name="gNameDL" Offset="0x1230" Ucode="f3dex2" RawPointers="0x08000000,0x09000000"/>
<DList Name="gNameDL" Offset="0x1230" Length="123" Ucode="f3dex2" RawPointers="0x08000000,0x09000000"/>
```
A display list.
- Optional attributes: `Ucode`, `RawPointers`
- Optional attributes: `Length`, `Ucode`, `RawPointers`
`Length` can be set to indicate the length (amount of `Gfx` double-words) of the dlist. If not set, the dlist length is automatic.
`Ucode` (defaults to `f3dex2`) picks the graphics microcode for which to disassemble the dlist. It may be `f3dex` or `f3dex2`.
@@ -97,6 +106,7 @@ A fixed-point matrix.
```xml
<Texture Name="gNameTex" Format="rgba16" Width="16" Height="16" Offset="0x1230"/>
<Texture Name="gNameTex" Format="ci8" Width="16" Height="16" Offset="0x1230" Tlut="gNameTLUT"/>
<Texture Name="gNameTex" Format="ci8" Width="16" Height="16" Offset="0x1230" TlutOffset="0x2340"/>
<Texture Name="gNameTex" Format="ci8" Width="16" Height="16" Offset="0x1230" ExternalTlut="baserom_file" ExternalTlutOffset="0x2340"/>
```
@@ -104,13 +114,13 @@ A fixed-point matrix.
A texture, an image in one of the native N64 formats.
- Required attributes for all formats: `Format`, `Width`, `Height`
- Required attributes for CI formats (`ci4`, `ci8`): `TlutOffset`, or `ExternalTlut` and `ExternalTlutOffset`
- Required attributes for CI formats (`ci4`, `ci8`): `Tlut`, or `TlutOffset`, or `ExternalTlut` and `ExternalTlutOffset`
`Format` is the format of the texture, one of `rgba32`, `rgba16`, `i4`, `i8`, `ia4`, `ia8`, `ia16`, `ci4` or `ci8`.
`Width` and `Height` specify the dimensions of the texture.
For CI formats, the TLUT (Texture Look Up Table, or palette) must be specified with either `TlutOffset` if the TLUT is in the same file as the texture, or both of `ExternalTlut` and `ExternalTlutOffset` if the TLUT is in a different file. `ExternalTlut` is the name of the baserom file where the TLUT is. In both cases, the TLUT must also be declared as a resource.
For CI formats, the TLUT (Texture Look Up Table, or palette) must be specified with either `Tlut` or `TlutOffset` if the TLUT is in the same file as the texture, or both of `ExternalTlut` and `ExternalTlutOffset` if the TLUT is in a different file. `ExternalTlut` is the name of the baserom file where the TLUT is. In both cases, the TLUT must also be declared as a resource.
## `Array`
+39 -16
View File
@@ -10,27 +10,30 @@ from .base import (
ResourceDesc,
ResourcesDescCollection,
ResourceHandlerNeedsPass2Exception,
ResourceHasNoSizeError,
)
from . import xml_errors
@dataclasses.dataclass(eq=False)
class CollisionResourceDesc(ResourceDesc):
pass
def get_size(self):
return 0x2C
def handler_Collision(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset"})
xml_errors.check_attrib(reselem, {"Name"}, {"Offset"})
return CollisionResourceDesc(symbol_name, offset, collection, reselem)
@dataclasses.dataclass(eq=False)
class AnimationResourceDesc(ResourceDesc):
pass
def get_size(self):
return 0x10
def handler_Animation(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset"})
xml_errors.check_attrib(reselem, {"Name"}, {"Offset"})
return AnimationResourceDesc(symbol_name, offset, collection, reselem)
@@ -40,7 +43,7 @@ class PlayerAnimationResourceDesc(ResourceDesc):
def handler_PlayerAnimation(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset"})
xml_errors.check_attrib(reselem, {"Name"}, {"Offset"})
return PlayerAnimationResourceDesc(symbol_name, offset, collection, reselem)
@@ -50,7 +53,7 @@ class LegacyAnimationResourceDesc(ResourceDesc):
def handler_LegacyAnimation(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset"})
xml_errors.check_attrib(reselem, {"Name"}, {"Offset"})
return LegacyAnimationResourceDesc(symbol_name, offset, collection, reselem)
@@ -60,7 +63,7 @@ class CutsceneResourceDesc(ResourceDesc):
def handler_Cutscene(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset"})
xml_errors.check_attrib(reselem, {"Name"}, {"Offset"})
return CutsceneResourceDesc(symbol_name, offset, collection, reselem)
@@ -70,7 +73,7 @@ class SceneResourceDesc(ResourceDesc):
def handler_Scene(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset"})
xml_errors.check_attrib(reselem, {"Name"}, {"Offset"})
return SceneResourceDesc(symbol_name, offset, collection, reselem)
@@ -80,7 +83,7 @@ class RoomResourceDesc(ResourceDesc):
def handler_Room(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset"}, {"HackMode"})
xml_errors.check_attrib(reselem, {"Name"}, {"Offset", "HackMode"})
res = RoomResourceDesc(symbol_name, offset, collection, reselem)
if reselem.attrib.get("HackMode") == "syotes_room":
res.hack_modes.add("hackmode_syotes_room")
@@ -93,7 +96,7 @@ class PlayerAnimationDataResourceDesc(ResourceDesc):
def handler_PlayerAnimationData(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset", "FrameCount"})
xml_errors.check_attrib(reselem, {"Name", "FrameCount"}, {"Offset"})
frame_count = int(reselem.attrib["FrameCount"])
return PlayerAnimationDataResourceDesc(
symbol_name, offset, collection, reselem, frame_count
@@ -106,7 +109,7 @@ class PathListResourceDesc(ResourceDesc):
def handler_PathList(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset", "NumPaths"})
xml_errors.check_attrib(reselem, {"Name", "NumPaths"}, {"Offset"})
num_paths = int(reselem.attrib["NumPaths"])
return PathListResourceDesc(symbol_name, offset, collection, reselem, num_paths)
@@ -133,12 +136,21 @@ class SkeletonResourceDesc(ResourceDesc):
limb_enum_none_member_name: Optional[str]
limb_enum_max_member_name: Optional[str]
def get_size(self):
skel_size = {
SkeletonType.NORMAL: 0x8,
SkeletonType.FLEX: 0xC,
}.get(self.type)
if skel_size is None:
raise ResourceHasNoSizeError()
return skel_size
def handler_Skeleton(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(
reselem,
{"Name", "Offset", "Type", "LimbType"},
{"EnumName", "LimbNone", "LimbMax"},
{"Name", "Type", "LimbType"},
{"Offset", "EnumName", "LimbNone", "LimbMax"},
)
skel_type = SkeletonType[reselem.attrib["Type"].upper()]
limb_type = LimbType[reselem.attrib["LimbType"].upper()]
@@ -160,9 +172,17 @@ class LimbResourceDesc(ResourceDesc):
limb_type: LimbType
limb_enum_member_name: Optional[str]
def get_size(self):
limb_size = {
LimbType.STANDARD: 0xC,
}.get(self.limb_type)
if limb_size is None:
raise ResourceHasNoSizeError()
return limb_size
def handler_Limb(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset", "LimbType"}, {"EnumName"})
xml_errors.check_attrib(reselem, {"Name", "LimbType"}, {"Offset", "EnumName"})
limb_type = LimbType[reselem.attrib["LimbType"].upper()]
return LimbResourceDesc(
symbol_name,
@@ -179,9 +199,12 @@ class LimbTableResourceDesc(ResourceDesc):
limb_type: LimbType
count: int
def get_size(self):
return self.count * 4
def handler_LimbTable(symbol_name, offset, collection, reselem: Element):
xml_errors.check_attrib(reselem, {"Name", "Offset", "LimbType", "Count"})
xml_errors.check_attrib(reselem, {"Name", "LimbType", "Count"}, {"Offset"})
limb_type = LimbType[reselem.attrib["LimbType"].upper()]
count = int(reselem.attrib["Count"])
return LimbTableResourceDesc(
@@ -197,7 +220,7 @@ class CurveAnimationResourceDesc(ResourceDesc):
def handler_CurveAnimation(
symbol_name, offset, collection: ResourcesDescCollection, reselem: Element
):
xml_errors.check_attrib(reselem, {"Name", "Offset", "SkelOffset"})
xml_errors.check_attrib(reselem, {"Name", "SkelOffset"}, {"Offset"})
res = CurveAnimationResourceDesc(symbol_name, offset, collection, reselem, None)
skel_offset = int(reselem.attrib["SkelOffset"], 16)
@@ -1223,6 +1223,12 @@ class DListResource(Resource, can_size_be_unknown=True):
self.target_ucode = target_ucode
self.ignored_raw_pointers: set[int] = set()
def set_length(self, length: int):
if self.range_end is not None:
if length != ((self.range_end - self.range_start) // 8):
raise ValueError("length already set and different")
self.range_end = self.range_start + length * 8
def try_parse_data(self, memory_context):
offset = self.range_start
@@ -1340,8 +1346,13 @@ class DListResource(Resource, can_size_be_unknown=True):
return pygfxd.gfxd_macro_dflt()
if self.range_end is None:
dlist_data = self.file.data[self.range_start :]
else:
dlist_data = self.file.data[self.range_start : self.range_end]
size = gfxdis(
input_buffer=self.file.data[self.range_start :],
input_buffer=dlist_data,
target=self.target_ucode.gfxd_ucode,
vtx_callback=vtx_cb,
timg_callback=timg_cb,
@@ -229,6 +229,8 @@ def register_resource_handlers():
n64resources.GfxMicroCode.F3DEX2: dlist_resources.Ucode.f3dex2,
}[resource_desc.ucode],
)
if resource_desc.length is not None:
res.set_length(resource_desc.length)
res.ignored_raw_pointers |= resource_desc.raw_pointers
return res