Add extractors for some assets

tools/asset_extractor/.gitignore

@@ -1 +1,2 @@
-!asset_extractor.py
+!asset_extractor.py
+!assets/*.py

tools/asset_extractor/asset_extractor.py

@@ -2,10 +2,15 @@ from pathlib import Path
 import os
 import sys
 import subprocess
-import yaml
 from distutils.util import strtobool
 import json
 
+from assets.palette_group import PaletteGroup
+from assets.gfx_group import GfxGroup
+from assets.fixed_type_gfx import FixedTypeGfx
+from assets.frame_obj_lists import FrameObjLists
+from assets.extra_frame_offsets import ExtraFrameOffsets
+
 verbose = False
 
 def extract_assets(variant, assets_folder):
@@ -26,12 +31,12 @@ def extract_assets(variant, assets_folder):
     baserom_path = map[variant]
     with open(baserom_path, 'rb') as file:
         baserom = bytearray(file.read())
-    
-    config_modified = os.path.getmtime('assets.yaml')
-    json_modified = os.path.getmtime('assets.json')
-    if json_modified < config_modified:
-        print('Convert yaml to json...', flush=True)
-        subprocess.check_call('cat assets.yaml | yq . > assets.json', shell=True)
+
+    config_modified = os.path.getmtime('assets.json')
+    # json_modified = os.path.getmtime('assets.json')
+    # if json_modified < config_modified:
+    #     print('Convert yaml to json...', flush=True)
+    #     subprocess.check_call('cat assets.yaml | yq . > assets.json', shell=True)
 
     with open('assets.json') as file:
         current_offset = 0
@@ -68,7 +73,7 @@ def extract_assets(variant, assets_folder):
                     if verbose:
                         print(f'{path} does not yet exist.')
                     extract_file = True
-                        
+
 
                 if extract_file:
                     if verbose:
@@ -89,7 +94,7 @@ def extract_assets(variant, assets_folder):
 
                     if 'size' in asset: # The asset has a size and want to be extracted first.
                         size = asset['size'] # TODO can different sizes for the different variants ever occur?
-    
+
                         with open(path, 'wb') as output:
                             output.write(baserom[start:start+size])
                     # If an asset has no size, the extraction tool reads the baserom iself.
@@ -106,6 +111,24 @@ def extract_assets(variant, assets_folder):
                         extract_midi(path, baserom_path, start, options)
                     elif mode == 'aif':
                         extract_aif(path, options)
+                    elif mode == 'palette_group':
+                        palette_group = PaletteGroup(path, start, size, options)
+                        palette_group.extract_binary(baserom)
+                    elif mode == 'gfx_group':
+                        gfx_group = GfxGroup(path, start, size, options)
+                        gfx_group.extract_binary(baserom)
+                    elif mode == 'fixed_type_gfx':
+                        fixed_type_gfx = FixedTypeGfx(path, start, size, options)
+                        fixed_type_gfx.extract_binary(baserom)
+                    elif mode == 'frame_obj_lists':
+                        frame_obj_lists = FrameObjLists(path, start, size, options)
+                        frame_obj_lists.extract_binary(baserom)
+                    elif mode == 'extra_frame_offsets':
+                        extra_frame_offsets = ExtraFrameOffsets(path, start, size, options)
+                        extra_frame_offsets.extract_binary(baserom)
+                    elif mode != '':
+                        print(f'Asset type {mode} not yet implemented')
+
 
 
 
@@ -168,7 +191,7 @@ def extract_midi(path, baserom_path, start, options):
                     agb2mid_params.append('-t')
                     agb2mid_params.append(str(change['nominator']))
                     agb2mid_params.append(str(change['denominator']))
-                    agb2mid_params.append(str(change['time']))    
+                    agb2mid_params.append(str(change['time']))
             else:
                 agb2mid_params.append('-t')
                 agb2mid_params.append(str(changes['nominator']))

tools/asset_extractor/assets/base.py

@@ -0,0 +1,51 @@
+ROM_OFFSET = 0x8000000
+
+class BaseAsset:
+    def __init__(self, path: str, addr: int, size: int, options: any) -> None:
+        self.path = path
+        self.addr = addr
+        self.size = size
+        self.options = options
+
+    def extract_binary(self, rom: bytearray) -> None:
+        pass
+
+    def convert(self) -> None:
+        pass
+
+class Reader:
+    def __init__(self, data: bytearray) -> None:
+        self.data = data
+        self.cursor = 0
+        self.bitfield = 0
+        self.bitfield_remaining = 0
+
+    def read_u8(self) -> int:
+        val = self.data[self.cursor]
+        self.cursor += 1
+        return val
+
+    def read_s8(self) -> int:
+        val = self.data[self.cursor]
+        self.cursor += 1
+        if val > 127:
+            return val-256
+        else:
+            return val
+
+    def read_u16(self) -> int:
+        val = self.data[self.cursor:self.cursor+2]
+        self.cursor += 2
+        return int.from_bytes(val, 'little')
+
+    def read_s16(self) -> int:
+        val = self.read_u16()
+        if val > 32768:
+            return val - 65536
+        else:
+            return val
+
+    def read_u32(self) -> int:
+        val = self.data[self.cursor:self.cursor+4]
+        self.cursor += 4
+        return int.from_bytes(val, 'little')

tools/asset_extractor/assets/extra_frame_offsets.py

@@ -0,0 +1,76 @@
+from assets.base import BaseAsset, Reader
+
+class ExtraFrameOffsets(BaseAsset):
+    def __init__(self, path: str, addr: int, size: int, options: any) -> None:
+        super().__init__(path, addr, size, options)
+
+    def extract_binary(self, rom: bytearray) -> None:
+        reader = Reader(rom[self.addr:self.addr+self.size])
+
+        first_level = []
+        second_level = []
+
+        lines = []
+        bytes = []
+        for i in range(0x10):
+            bytes.append(reader.read_u8())
+        lines.append('\t.byte ' + ', '.join(str(x) for x in bytes) + '\n')
+
+        lines.append('@ First level of offsets\n')
+
+        while True:
+            if reader.cursor in first_level:
+                #print(f'first_level up to: {reader.cursor}')
+                break
+            pointer = reader.read_u16()
+            first_level.append(pointer)
+            lines.append(f'\t.2byte {hex(pointer)}\n')
+
+        #print(first_level)
+        #print(first_level)
+        lines.append('\n@ Second level of offsets\n')
+        while True:
+            #print(reader.cursor)
+            #if reader.cursor >= 24372:
+                #print(f'>< second_level up to: {reader.cursor}')
+                #
+                # break
+            if reader.cursor >= 0xD00:
+                #print(f'second_level up to: {reader.cursor}')
+                break
+            pointer = reader.read_u8()
+            second_level.append(pointer)
+            lines.append(f'\t.byte {hex(pointer)}\n')
+        obj_lists = []
+        lines.append('\n@ Extra frame offsets\n')
+        while True:
+            #print('WH')
+            if (reader.cursor-0xD00)/4 not in second_level:
+                #print(f'{reader.cursor} not in second_level')
+                break
+                next = -1
+                for i in second_level:
+                    if i > reader.cursor:
+                        if next == -1 or i < next:
+                            next = i
+
+                diff = next-reader.cursor
+                print(f'Skipping forward to {next} (+{diff})')
+                lines.append(f'@ Skipping {diff} bytes\n')
+                bytes = []
+                for i in range(diff):
+                    bytes.append(reader.read_u8())
+                lines.append('\t.byte ' + ', '.join(str(x) for x in bytes) + '\n')
+
+            extra_x_off = reader.read_s8()
+            extra_y_off = reader.read_s8()
+            lines.append(f'\t.byte {extra_x_off}, {extra_y_off}\n')
+
+            extra_x_off = reader.read_s8()
+            extra_y_off = reader.read_s8()
+            lines.append(f'\t.byte {extra_x_off}, {extra_y_off}\n')
+
+        assert(self.path.endswith('.bin'))
+        path = self.path[0:-4] + '.s'
+        with open(path, 'w') as file:
+            file.writelines(lines)

tools/asset_extractor/assets/fixed_type_gfx.py

@@ -0,0 +1,26 @@
+from assets.base import BaseAsset, Reader
+
+class FixedTypeGfx(BaseAsset):
+    def __init__(self, path: str, addr: int, size: int, options: any) -> None:
+        super().__init__(path, addr, size, options)
+
+    def extract_binary(self, rom: bytearray) -> None:
+        reader = Reader(rom[self.addr:self.addr+self.size])
+        lines = []
+
+        index = 0
+        while reader.cursor < self.size:
+            pointer = reader.read_u32()
+            gfx_data_ptr = pointer& 0x00FFFFFC
+            compressed = pointer& 0x00000001
+
+            maybe_size = ((pointer >> 0x10) & 0x7f00) >> 4
+
+            gfx_data_len = ((pointer & 0x7F000000)>>24) * 0x200
+            lines.append(f'\t.4byte {hex(gfx_data_ptr)} + {compressed} + {hex((gfx_data_len//0x200))}<<24  @{index}\n')
+            index += 1
+
+        assert(self.path.endswith('.bin'))
+        path = self.path[0:-4] + '.s'
+        with open(path, 'w') as file:
+            file.writelines(lines)

tools/asset_extractor/assets/frame_obj_lists.py

@@ -0,0 +1,101 @@
+from assets.base import BaseAsset, Reader
+
+class FrameObjLists(BaseAsset):
+    def __init__(self, path: str, addr: int, size: int, options: any) -> None:
+        super().__init__(path, addr, size, options)
+
+    def extract_binary(self, rom: bytearray) -> None:
+        reader = Reader(rom[self.addr:self.addr+self.size])
+
+        first_level = []
+        second_level = []
+
+        lines = []
+        lines.append('@ First level of offsets\n')
+        while True:
+            if reader.cursor in first_level:
+                #print(f'first_level up to: {reader.cursor}')
+                break
+            pointer = reader.read_u32()
+            first_level.append(pointer)
+            lines.append(f'\t.4byte {hex(pointer)}\n')
+
+        #print(first_level)
+        lines.append('\n@ Second level of offsets\n')
+        while True:
+            #print(reader.cursor)
+            #if reader.cursor >= 24372:
+                #print(f'>< second_level up to: {reader.cursor}')
+                #
+                # break
+            if reader.cursor in second_level:
+                #print(f'second_level up to: {reader.cursor}')
+                break
+            pointer = reader.read_u32()
+            second_level.append(pointer)
+            lines.append(f'\t.4byte {hex(pointer)}\n')
+        #print(second_level)
+
+        obj_lists = []
+        last_second_level = max(second_level)
+        lines.append('\n@ Frame obj lists\n')
+        while True:
+            if reader.cursor > last_second_level:
+                #print(f'No longer in second level: {reader.cursor}')
+                break
+            if reader.cursor not in second_level:
+                #print(f'{reader.cursor} not in second_level {num_objects}')
+                next = -1
+                for i in second_level:
+                    if i > reader.cursor:
+                        if next == -1 or i < next:
+                            next = i
+
+                diff = next-reader.cursor
+                #print(f'Skipping forward to {next} (+{diff})')
+                lines.append(f'@ Skipping {diff} bytes\n')
+                bytes = []
+                for i in range(diff):
+                    bytes.append(reader.read_u8())
+                lines.append('\t.byte ' + ', '.join(str(x) for x in bytes) + '\n')
+            num_objects = reader.read_u8()
+            lines.append(f'\t.byte {num_objects}\n')
+            if num_objects > 200:
+                #print(f'num_objects: {num_objects} @{reader.cursor}/{last_second_level}')
+                break
+            list = []
+            #print(num_objects)
+            for i in range(num_objects):
+                x_offset = reader.read_s8()
+                y_offset = reader.read_s8()
+                bitfield = reader.read_u8()
+                bitfield2 = reader.read_u16()
+
+                lines.append(f'\t.byte {x_offset}, {y_offset}, {hex(bitfield)}\n')
+                lines.append(f'\t.2byte {hex(bitfield2)}\n')
+
+                # bitfield
+                override_entity_palette_index = (bitfield & 0x01) != 0
+                # Bit 02 seems unused.
+                h_flip = (bitfield & 0x04) != 0
+                v_flip = (bitfield & 0x08) != 0
+                size = (bitfield & 0x30) >> 4
+                shape = (bitfield & 0xC0) >> 6
+
+                # bitfield2
+                first_gfx_tile_offset = bitfield2 & 0x03FF
+                priority = (bitfield2 & 0x0C00) >> 10
+                palette_index = (bitfield2 & 0xF000) >> 12
+
+
+                # print(x_offset, y_offset, bitfield, bitfield2)
+                # print(override_entity_palette_index, h_flip, v_flip, size, shape)
+                # print(first_gfx_tile_offset, priority, palette_index)
+                list.append({})
+                # print()
+            obj_lists.append(list)
+        #print(len(obj_lists))
+        assert(self.path.endswith('.bin'))
+        path = self.path[0:-4] + '.s'
+        with open(path, 'w') as file:
+            file.writelines(lines)

tools/asset_extractor/assets/gfx_group.py

@@ -0,0 +1,31 @@
+from assets.base import BaseAsset, Reader
+
+class GfxGroup(BaseAsset):
+    def __init__(self, path: str, addr: int, size: int, options: any) -> None:
+        super().__init__(path, addr, size, options)
+
+    def extract_binary(self, rom: bytearray) -> None:
+        reader = Reader(rom[self.addr:self.addr+self.size])
+        gfx_index = 0
+        lines = []
+        while reader.cursor < self.size:
+            unk0 = reader.read_u32()
+            gfx_offset = unk0 & 0xFFFFFF
+            dest = reader.read_u32()
+            unk8 = reader.read_u32()
+            size = unk8 & 0xFFFFFF
+            terminator = unk0 & 0x80000000
+
+            compressed = unk8 & 0x80000000
+            uncompressed_size = size
+
+
+            lines.append(f'\t.4byte {hex(gfx_offset)}+{hex(terminator)}+{hex(unk0 & 0xF000000)}, {hex(dest)}, {hex(uncompressed_size)} + {hex(compressed)} @ {gfx_index}\n')
+            if not terminator:
+                break
+            gfx_index += 1
+
+        assert(self.path.endswith('.bin'))
+        path = self.path[0:-4] + '.s'
+        with open(path, 'w') as file:
+            file.writelines(lines)

tools/asset_extractor/assets/palette_group.py

@@ -0,0 +1,36 @@
+from assets.base import ROM_OFFSET, BaseAsset, Reader
+
+class PaletteGroup(BaseAsset):
+    def __init__(self, path: str, addr: int, size: int, options: any) -> None:
+        super().__init__(path, addr, size, options)
+
+    def extract_binary(self, rom: bytearray) -> None:
+        self.extract_palette_group(rom, self.addr, self.size)
+        #self.extract_palette_groups(rom)
+
+    def extract_palette_group(self, rom: bytearray, address: int, size: int) -> tuple[list[str], list[int]]:
+        lines: list[str] = []
+        palette_indices: list[int] = []
+
+        assert(self.path.endswith('.bin'))
+        path = self.path[0:-4] + '.s'
+
+        reader = Reader(rom[address:address+size])
+        continue_loading_palette_sets = True
+        while continue_loading_palette_sets:
+            global_palette_index = reader.read_u16()
+            palette_load_offset = reader.read_u8()
+            bitfield = reader.read_u8()
+
+            num_palettes = bitfield & 0x0F
+            if num_palettes == 0:
+                num_palettes = 0x10
+            continue_loading_palette_sets = (bitfield & 0x80 == 0x80)
+            lines.append(f'\t.2byte {global_palette_index} @ palette index\n')
+            lines.append(f'\t.byte {palette_load_offset} @ palette offset\n')
+            lines.append(f'\t.byte {num_palettes if num_palettes < 0x10 else 0} + {continue_loading_palette_sets*0x80} @ num_palettes, continue_loading\n\n')
+            for i in range(num_palettes):
+                palette_indices.append(global_palette_index + i)
+
+        with open(path, 'w') as file:
+            file.writelines(lines)