mirror/tp
1
0
Fork 0
mirror of https://github.com/zeldaret/tp synced 2026-09-05 02:36:01 -04:00
Code Issues Packages Projects Releases Wiki Activity

moved strings + decompile simple store functions

Browse Source
This commit is contained in:
Julgodis
2021-04-10 07:02:38 +02:00
parent 2661db6aaa
commit 0ad6ebe7b4
1299 changed files with 51830 additions and 43634 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/libdol2asm/data/base.py
+1 -6
View File
@@ -215,12 +215,7 @@ class ArbitraryData(Symbol):
await self.export_declaration_body(exporter, builder)
if self._section == ".rodata":
await builder.write(f"COMPILER_STRIP_GATE({self.addr:08X}, {self.cpp_reference(None, self.addr)});")
# await builder.write_nonewline("SECTION_DEAD ")
# await builder.write_nonewline("void* const ")
# await builder.write_nonewline(f"cg_{self.addr:08X} = (void*)(")
# await builder.write_nonewline(self.cpp_reference(None, self.addr))
# await builder.write(f");")
await builder.write(f"COMPILER_STRIP_GATE(0x{self.addr:08X}, {self.cpp_reference(None, self.addr)});")
if self.requires_force_active:
await builder.write(f"#pragma pop")
tools/libdol2asm/data/function/__init__.py
+2
View File
@@ -3,3 +3,5 @@ from .base import *
from .asm import *
from .ret import *
from .sinit import *
from .store import *
from .small_asm import *
tools/libdol2asm/data/function/base.py
+11 -4
View File
@@ -132,12 +132,19 @@ class Function(Symbol):
for line in lines:
await builder.write(line)
arg_type = ""
arg_type = []
if forward:
arg_type = ", ".join([x.type(specialize_templates=True) for x in self.argument_types])
arg_type = [x.type(specialize_templates=True) for x in self.argument_types]
else:
arg_type = ", ".join([x.decl(f"param_{i}",specialize_templates=True) for i, x in zip(
range(len(self.argument_types)), self.argument_types)])
arg_type = [
x.decl(f"param_{i}",specialize_templates=True)
for i, x in enumerate(self.argument_types)
]
if self.demangled_name and self.demangled_name.require_specialization:
arg_type = [f"void* _this", *arg_type]
arg_type = ", ".join(arg_type)
if self._section == ".init":
await builder.write_nonewline(f"SECTION_INIT ")
tools/libdol2asm/data/function/ret.py
+7 -2
View File
@@ -29,6 +29,8 @@ class CustomReturnFunction(ReturnFunction):
class SymbolReturnFunction(ReturnFunction):
symbol_addr: int = 0
load_or_reference: bool = True
load_type: Type = None
cast_type: Type = None
def gather_references(self, context, valid_range):
self.references = set([ self.symbol_addr ])
@@ -37,8 +39,11 @@ class SymbolReturnFunction(ReturnFunction):
symbol = symbol_table[-1, self.symbol_addr]
assert symbol
name = symbol.cpp_reference(self, self.symbol_addr)
type = PointerType(self.return_type)
load_type = PointerType(self.load_type)
dereference = ""
cast = ""
if self.load_or_reference:
dereference = "*"
return f"{dereference}({type.type()})({name})"
if self.cast_type:
cast = f"({self.cast_type.type()})"
return f"{cast}{dereference}({load_type.type()})({name})"
tools/libdol2asm/data/function/sinit.py
+1 -1
View File
@@ -20,7 +20,7 @@ class SInitFunction(ASMFunction):
await builder.write("#pragma push")
await builder.write("#pragma force_active on")
await builder.write(f"SECTION_CTORS void* const _ctors_{self.addr:08X} = (void*){self.label};")
await builder.write(f"REGISTER_CTORS(0x{self.addr:08X}, {self.label});")
await builder.write("#pragma pop")
await builder.write("")
tools/libdol2asm/data/function/small_asm.py
+20
View File
@@ -0,0 +1,20 @@
from dataclasses import dataclass, field
from typing import List
from ...builder import AsyncBuilder
from .base import *
@dataclass(eq=False)
class SmallASMFunction(Function):
asm: bool = True
insts: List[str] = field(default_factory=list)
async def export_function_body(self, exporter, builder: AsyncBuilder):
await builder.write(f" {{")
await builder.write(f"\t// clang-format off")
await builder.write(f"\tnofralloc")
for inst in self.insts:
await builder.write(f"\t{inst}")
await builder.write(f"\t// clang-format on")
await builder.write(f"}}")
tools/libdol2asm/data/function/store.py
+55
View File
@@ -0,0 +1,55 @@
from dataclasses import dataclass, field
from capstone import *
from capstone.ppc import *
from ...builder import AsyncBuilder
from ...types import *
from .base import *
@dataclass(eq=False)
class StoreFunction(Function):
async def export_store(self, exporter, builder):
assert False
async def export_function_body(self, exporter, builder: AsyncBuilder):
await builder.write(f" {{")
await self.export_store(exporter, builder)
await builder.write(f"}}")
@dataclass(eq=False)
class Store_R3_OffsetRX_Function(StoreFunction):
dst: int = 0
dst_offset: int = 0
src: int = 0
store_type: Type = None
def calculate_params(self):
params = {}
gr = PPC_REG_R3
if self.has_class:
if self.demangled_name and self.demangled_name.require_specialization:
params[gr] = "_this"
else:
params[gr] = "this"
gr += 1
for i, arg in enumerate(self.argument_types):
params[gr] = f"param_{i}"
gr += 1
return params
async def export_store(self, exporter, builder):
params = self.calculate_params()
dst = params[self.dst]
src = params[self.src]
if self.dst_offset > 0:
dst = f"(((u8*){params[self.dst]})+{self.dst_offset}) /* {params[self.dst]}->field_0x{self.dst_offset:x} */"
pointer_type = PointerType(self.store_type)
await builder.write(f"\t*({pointer_type.type()}){dst} = ({self.store_type.type()})({src});")
tools/libdol2asm/data/string.py
+31 -3
View File
@@ -58,13 +58,27 @@ def escape_full_string(data):
class String(ArbitraryData):
encoding: str = None
decoded_string: str = None
string_base: "StringBase" = None
def array_type(self):
return self.element_type()
async def export_declaration(self, exporter, builder: AsyncBuilder):
assert self.padding == 0
def asm_reference(self, addr):
if self.string_base:
return self.string_base.asm_reference(addr)
else:
return super().asm_reference(addr)
def cpp_reference(self, accessor, addr):
if self.string_base:
return self.string_base.cpp_reference(accessor, addr)
else:
return super().cpp_reference(accessor, addr)
async def export_declaration(self, exporter, builder: AsyncBuilder, force_active=True):
if force_active:
await builder.write("#pragma push")
await builder.write("#pragma force_active on")
sjis = self.decoded_string.encode("shift_jisx0213")
if 0x5c in sjis:
await builder.write("// MWCC ignores mapping of some japanese characters using the ")
@@ -79,6 +93,18 @@ class String(ArbitraryData):
await builder.write_nonewline(self.array_type().decl(self.identifier.label))
await String.export_string(builder, data)
if self.padding > 0:
assert len(self.padding_data) == self.padding
assert self.padding_data[-1] == 0
data = escape_full_string(self.padding_data[:-1])
await builder.write("/* @stringBase0 padding */")
await builder.write_nonewline("SECTION_DEAD ")
await builder.write_nonewline("static ")
await builder.write_nonewline(self.array_type().decl(f"pad_{self.end:08X}"))
await String.export_string(builder, data)
if force_active:
await builder.write("#pragma pop")
@staticmethod
async def export_string(builder: AsyncBuilder, data: List[str]):
if len(data) < 32:
@@ -119,9 +145,10 @@ class StringBase(ArbitraryData):
string.set_mlts(module, library, translation_unit, section)
async def export_declaration(self, exporter, builder: AsyncBuilder):
pass
"""
await builder.write("#pragma push")
await builder.write("#pragma force_active on")
await builder.write("#pragma section \".dead\"")
for string in self.strings:
# if the @stringBase0 is static (which it will almost always be), setup
# so that the sub-strings are static.
@@ -139,6 +166,7 @@ class StringBase(ArbitraryData):
await builder.write_nonewline(self.array_type().decl(f"pad_{self.end:08X}"))
await String.export_string(builder, data)
await builder.write("#pragma pop")
"""
@staticmethod
def create(symbol, strings, data, padding_data):
tools/libdol2asm/exporter/cpp_exporter.py
+33 -68
View File
@@ -8,6 +8,7 @@ from pathlib import Path
from collections import defaultdict
from typing import List, Dict
from dataclasses import dataclass, field
from itertools import groupby
from .. import util
from .. import settings
@@ -36,6 +37,7 @@ order = {
".init": 8
}
@dataclass
class CPPExporter:
context: Context
@@ -44,13 +46,13 @@ class CPPExporter:
async def export_symbol_header(self, builder: AsyncBuilder, symbol: Symbol):
await builder.write("/* %08X-%08X %06X %04X+%02X %i/%i %i/%i %i/%i %-16s %-60s */" % (
symbol.start, symbol.end+symbol.padding,
symbol.start, symbol.end+symbol.padding,
symbol.relative_addr,
symbol.size, symbol.padding,
symbol.reference_count.static,
symbol.implicit_reference_count.static,
symbol.reference_count.extern,
symbol.implicit_reference_count.extern,
symbol.reference_count.static,
symbol.implicit_reference_count.static,
symbol.reference_count.extern,
symbol.implicit_reference_count.extern,
symbol.reference_count.rel,
symbol.implicit_reference_count.rel,
symbol._section, symbol.identifier.name))
@@ -142,8 +144,6 @@ class CPPExporter:
else:
section.symbols.sort(key=lambda x: x.addr)
for function, symbols, forward_symbols in function_symbols_groups:
# new section of symbols followed by a function
if len(symbols) > 0:
@@ -156,11 +156,22 @@ class CPPExporter:
await builder.write("")
unreferenced_decls = 0
for symbol in symbols:
assert not isinstance(symbol, StringBase)
await self.export_symbol_header(builder, symbol)
await symbol.export_declaration(self, builder)
await builder.write("")
symbol_groups = [list(g) for k, g in groupby(symbols, key=lambda x: isinstance(x, String))]
for symbols in symbol_groups:
if isinstance(symbols[0], String):
await self.export_symbol_header(builder, symbols[0].string_base)
await builder.write("#pragma push")
await builder.write("#pragma force_active on")
for symbol in symbols:
await symbol.export_declaration(self, builder, force_active=False)
await builder.write("#pragma pop")
await builder.write("")
else:
for symbol in symbols:
assert not isinstance(symbol, StringBase)
await self.export_symbol_header(builder, symbol)
await symbol.export_declaration(self, builder)
await builder.write("")
await self.export_symbol_header(builder, function)
await function.export_declaration(self, builder)
@@ -184,7 +195,7 @@ class CPPExporter:
await symbol.export_declaration(self, builder)
await builder.write("")
def gather_function_groups(self, decl_references):
def gather_function_groups(self, decl_references):
sections = list(self.tu.sections.values())
sections.sort(key=lambda x: order[x.name]
if x.name in order else 10 + len(x.name))
@@ -229,10 +240,14 @@ class CPPExporter:
# add missing references so that the order is still correct
missing_order_symbols = []
for symbol in symbols:
is_string = isinstance(symbol, String)
symbol_section = self.tu.sections[symbol._section]
for prev_symbol in symbol_section.symbols:
if prev_symbol == symbol:
break
prev_is_string = isinstance(prev_symbol, String)
if is_string != prev_is_string:
continue
if isinstance(prev_symbol, Function) or isinstance(prev_symbol, StringBase):
continue
if prev_symbol in used_symbols:
@@ -301,10 +316,9 @@ class CPPExporter:
type_list.build(forward_references)
type_list.build(external_references)
already_fixed_forward_reference = set()
function_symbols_groups, fsg_used_symbols = self.gather_function_groups(decl_references)
function_symbols_groups, fsg_used_symbols = self.gather_function_groups(
decl_references)
for function, symbols, forward_symbols in function_symbols_groups:
for symbol in symbols:
if isinstance(symbol, StringBase):
@@ -313,14 +327,15 @@ class CPPExporter:
continue
already_fixed_forward_reference.add(symbol)
forward_references = list(decl_references - already_fixed_forward_reference)
forward_references = list(
decl_references - already_fixed_forward_reference)
forward_references.sort(key=lambda x: x.addr)
stringBases = set()
for decl in decl_references:
if isinstance(decl, StringBase):
stringBases.add(decl)
decl_references = decl_references - stringBases
async with AsyncBuilder(path) as builder:
@@ -376,52 +391,6 @@ class CPPExporter:
await self.export_declarations(builder, tu, decl_references, function_symbols_groups, fsg_used_symbols)
"""
# symbols that are in the .rodata (read-only data) section will be stripped by the compiler (not the linker)
# if they are not used. create a dead-symbol to fake the use of the read-only symbol. This will mess with the
# of the symbols. E.g.
# SECTION_RODATA static u32 const lit_4125 = 0x42A00000;
# SECTION_RODATA static u32 const lit_4218 = 0x43130000;
# SECTION_DEAD void* const cg_805AA484 = (void*)(&lit_4125);
# will output:
# lit_4218
# @stringBase0 (if it exists)
# lit_4125
# this is incorrect! solution is, if any read-only symbol requires the fake dead symbol trick, to do it for
# all read-only symbols.
rodata = []
dead_rodata = []
for decl in decl_references:
if decl._section != ".rodata":
continue
rodata.append(decl)
if not decl.requires_force_active:
continue
dead_rodata.append(decl)
if dead_rodata:
await builder.write("// ")
await builder.write("// Read-Only Compiler Gate:")
await builder.write("// ")
await builder.write("")
await builder.write("#pragma push")
await builder.write("#pragma force_active on")
rodata.sort(key=lambda x: x.addr)
for decl in rodata:
await builder.write_nonewline("SECTION_DEAD ")
await builder.write_nonewline("void* const ")
await builder.write_nonewline(f"cg_{decl.addr:08X} = (void*)(")
await builder.write_nonewline(decl.cpp_reference(None, decl.addr))
await builder.write(f");")
await builder.write("#pragma pop")
await builder.write("")
"""
for stringBase in stringBases:
await self.export_symbol_header(builder, stringBase)
await stringBase.export_declaration(self, builder)
@@ -437,8 +406,6 @@ def export_translation_unit_group(context: Context, tus: List[Tuple[TranslationU
]
async def wait_all():
# for task in async_tasks:
# await task
await asyncio.gather(*async_tasks)
asyncio.run(wait_all())
@@ -473,8 +440,6 @@ def export_function(context: Context, section: Section, functions: List[Symbol],
]
async def wait_all():
# for task in async_tasks:
# await task
await asyncio.gather(*async_tasks)
asyncio.run(wait_all())
tools/libdol2asm/generate_functions.py
+136 -13
View File
@@ -163,22 +163,42 @@ def is_load_global_function(data: bytearray) -> Tuple[bool, int, str, int]:
return False, None, None, None
# TODO: @!game move
R2_ADDR = 0x80459A00
R13_ADDR = 0x80458580
RETURN_SYMBOL_LOAD_INSTS = {
LOAD_INSTS = {
PPC_INS_LWZ,
PPC_INS_LHZ,
PPC_INS_LHA,
PPC_INS_LBZ,
}
RETURN_SYMBOL_TYPE = {
LOAD_TYPE = {
PPC_INS_LWZ: U32,
PPC_INS_LHZ: U16,
PPC_INS_LHA: S16,
PPC_INS_LBZ: U8,
}
LOAD_CAST_TYPE = {
PPC_INS_LHA: S32,
}
STORE_INSTS = {
PPC_INS_STW,
PPC_INS_STH,
PPC_INS_STB,
}
STORE_TYPE = {
PPC_INS_STW: U32,
PPC_INS_STH: U16,
PPC_INS_STB: U8,
}
def decompile_return_symbol_function(symbol, block, insts, symbol_table) -> Function:
if len(insts) != 2:
return None
@@ -188,9 +208,9 @@ def decompile_return_symbol_function(symbol, block, insts, symbol_table) -> Func
if ret.id != PPC_INS_BLR:
return None
if not load.id in RETURN_SYMBOL_LOAD_INSTS:
if not load.id in LOAD_INSTS:
return None
address = 0
mem_base = load.operands[1].mem.base
mem_disp = load.operands[1].mem.disp
@@ -209,15 +229,101 @@ def decompile_return_symbol_function(symbol, block, insts, symbol_table) -> Func
if isinstance(return_symbol, Structure):
return None
load_type = LOAD_TYPE[load.id]
cast_type = None
if load.id in LOAD_CAST_TYPE:
cast_type = LOAD_CAST_TYPE[load.id]
return_type = cast_type
if not return_type:
return_type = load_type
return SymbolReturnFunction(
symbol.identifier,
addr=symbol.addr,
size=symbol.size,
padding=symbol.padding,
alignment=0,
return_type=RETURN_SYMBOL_TYPE[load.id],
return_type=return_type,
load_type=load_type,
cast_type=cast_type,
symbol_addr=address)
SMALL_INST = {
PPC_INS_MFMSR,
PPC_INS_MTMSR,
PPC_INS_TWUI,
PPC_INS_MTSPR,
PPC_INS_MFSPR,
PPC_INS_MFTB,
PPC_INS_MTFSB1,
PPC_INS_SC,
}
def inst_to_string(insn):
if insn.id == PPC_INS_TWUI:
assert insn.operands[0].type == PPC_OP_REG
assert insn.operands[1].type == PPC_OP_IMM
rA = insn.reg_name(insn.operands[0].reg)
S = insn.operands[1].value.imm
insn_str = 'twi %i, %s, 0x%x' % (31, rA, S)
return insn_str
return f"{insn.mnemonic} {insn.op_str}"
def decompile_small_asm_function(symbol, block, insts, symbol_table) -> Function:
if len(insts) != 2:
return None
unknown = insts[0]
ret = insts[1]
if not ret or ret.id != PPC_INS_BLR:
return None
if not unknown or not unknown.id in SMALL_INST:
return None
return SmallASMFunction(
symbol.identifier,
addr=symbol.addr,
size=symbol.size,
padding=symbol.padding,
alignment=0,
return_type=VOID,
insts=[inst_to_string(insn) for insn in insts])
def decompile_store_param_function(symbol, block, insts, symbol_table) -> Function:
if len(insts) != 2:
return None
store = insts[0]
ret = insts[1]
if ret.id != PPC_INS_BLR:
return None
if not store.id in STORE_INSTS:
return None
src = store.operands[0].reg
dst = store.operands[1].mem.base
dst_offset = store.operands[1].mem.disp
if dst == PPC_REG_R3:
return Store_R3_OffsetRX_Function(
symbol.identifier,
addr=symbol.addr,
size=symbol.size,
padding=symbol.padding,
alignment=0,
return_type=VOID,
store_type=STORE_TYPE[store.id],
dst=dst,
dst_offset=dst_offset,
src=src)
return None
def from_group(section: Section, group: List[linker_map.Symbol]) -> Function:
"""
@@ -228,7 +334,7 @@ def from_group(section: Section, group: List[linker_map.Symbol]) -> Function:
if len(group) == 1:
block = group[0]
data = section.get_data(block.start, block.end)
if len(data) >= 4 and len(data) < 16:
if len(data) >= 4 and len(data) < 16:
insts = list(disassemble.cs.disasm(data, block.start))
if is_return_function(data):
@@ -271,6 +377,7 @@ def from_group(section: Section, group: List[linker_map.Symbol]) -> Function:
# the function was not decompilable
return [ASMFunction.create(section, group)]
def decompile_symbol(context, section, symbol, symbol_table, add_list, remove_list):
if not isinstance(symbol, ASMFunction):
return symbol
@@ -283,20 +390,36 @@ def decompile_symbol(context, section, symbol, symbol_table, add_list, remove_li
return symbol
insts = list(disassemble.cs.disasm(data, block.start))
function = decompile_return_symbol_function(symbol, block, insts, symbol_table)
function = decompile_return_symbol_function(
symbol, block, insts, symbol_table)
if not function:
function = decompile_store_param_function(
symbol, block, insts, symbol_table)
if not function:
function = decompile_small_asm_function(
symbol, block, insts, symbol_table)
if len(data) == 8 and not function and insts[1] and insts[1].id == PPC_INS_BLR:
print(symbol.identifier)
for inst in insts:
print(f"\t{inst.mnemonic} {inst.op_str}")
if function:
function.set_mlts(symbol._module, symbol._library, symbol._translation_unit, symbol._section)
function.alignment = symbol.alignment
function.set_mlts(symbol._module, symbol._library,
symbol._translation_unit, symbol._section)
assert function.addr == symbol.addr
assert function.size == symbol.size
add_list.add(function)
remove_list.discard(symbol)
return function
#if symbol.size == 8:
# context.debug(f"{symbol.addr:08X} {symbol.name}")
if symbol.size == 8:
context.debug(f"{symbol.addr:08X} {symbol.label}")
return symbol
def decompile(context, libraries, symbol_table):
remove_list = set()
add_list = set()
@@ -307,7 +430,7 @@ def decompile(context, libraries, symbol_table):
continue
symbols = []
for symbol in section.symbols:
symbols.append(decompile_symbol(context, section, symbol, symbol_table, add_list, remove_list))
symbols.append(decompile_symbol(
context, section, symbol, symbol_table, add_list, remove_list))
section.symbols = symbols
return add_list, remove_list
tools/libdol2asm/generate_symbols.py
+25 -6
View File
@@ -24,7 +24,7 @@ def string_decode(data: bytearray) -> Tuple[str, str]:
return None, None
def string_from_data(addr: int, data: bytearray) -> String:
def string_from_data(addr: int, data: bytearray, string_base: StringBase) -> String:
""" Create string symbol from an address and data """
string, encoding = string_decode(data)
@@ -36,7 +36,8 @@ def string_from_data(addr: int, data: bytearray) -> String:
len(data),
data_type=PointerType(ConstType(CHAR)),
encoding=encoding,
decoded_string=string)
decoded_string=string,
string_base=string_base)
def zero_initialized_symbol(section: Section,
@@ -81,14 +82,32 @@ def value_initialized_symbol(section: Section,
# strings will always be in rodata
if section.name == ".rodata":
if symbol.name == "@stringBase0":
strings = []
string_base = StringBase(
Identifier("stringBase", symbol.addr, symbol.name),
symbol.addr,
0,
data = bytes(),
data_type=PointerType(ConstType(CHAR)),
padding=0,
padding_data=bytes(),
strings = [])
strings = [ string_base ]
split_data = list(util.magicsplit(data, 0))
x_offset = 0
for x in split_data[:-1]:
str_addr = symbol.addr + x_offset
str_length = len(x) + 1
str_data = bytes(x + [0])
strings.append(string_from_data(
symbol.addr + x_offset, bytes(x + [0])))
x_offset += len(x) + 1
return [StringBase.create(symbol, strings, data, padding_data)]
str_addr, str_data, string_base))
x_offset += str_length
#return [StringBase.create(symbol, strings, data, padding_data)]
strings[-1].padding = len(padding_data)
strings[-1].padding_data = padding_data
return strings
if section.name == ".init":
if symbol.name == "_rom_copy_info" or symbol.name == "_bss_init_info":
tools/libdol2asm/split_asm.py
-1
View File
@@ -664,7 +664,6 @@ class Dol2AsmSplitter:
self.search_binary(cache)
cache = True
start_time = time.time()
cache_path = Path("build/full_cache_xx.dump")
if cache and cache_path.exists():
tools/libdol2asm/tools.py
+1 -1
View File
@@ -87,7 +87,7 @@ def merge_section_symbols(context, section, add_list, remove_list):
symbols.extend(group)
for old_symbol in section.symbols:
is_unaligned = isinstance(old_symbol, ArbitraryData) and old_symbol.addr % 4 != 0
is_unaligned = type(old_symbol).__name__ == "ArbitraryData" and old_symbol.addr % 4 != 0
if is_unaligned:
assert group
tools/libdol2asm/types/function.py
+1 -1
View File
@@ -48,4 +48,4 @@ class FunctionType(Type):
without_template=without_template)
args = ", ".join([x.type(specialize_templates=specialize_templates,
without_template=without_template) for x in self.argument_types])
return f"{return_type} ({class_name}{inner_type})({args})"
return f"{return_type} ({class_name}{inner_type}{label})({args})"