mirror/tp
1
0
Fork 0
mirror of https://github.com/zeldaret/tp synced 2026-07-07 14:13:27 -04:00
Code Issues Packages Projects Releases Wiki Activity

Clean up and improvements to tools (#163)

Browse Source 
* moved elf2dol

* removed postprocess.py

* removed vtables.py

* find_unused_asm.py

* removed section2cpp.py

* removed splitter/*

* fixed symbol names due to iconv file rename

* fixed problem building RELs caused by #160

* improved performance of a few python tools

* added new tool for finding conflict when not OK

* added ./tp setup

* don't install dol2asm dependecies with requirements.txt

* format and check for imports

* remove unused tools/difftools.py

* fixed ignore to include elf2dol

* fix compiler patcher

* ok-check now creates the patched compiler at mwcceppc_patched.exe

* Add new command to copy the build folder to the expected folder

* 'make clean' will now only clean main.dol stuff. (added clean_rels and clean_all)

* './tp pull-request' and './tp check' now doesn't include RELs by default. Use '--rels' to include them in the process.

* './tp remove-unused-asm --check' added, exitcode 0==no files, 1==exists files

Co-authored-by: Julgodis <>
This commit is contained in:
Jonathan Wase
2021-12-02 23:38:37 +01:00
committed by GitHub
parent 5f187a0776
commit bc428f7f65
829 changed files with 5043 additions and 4328 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/.gitignore
+1 -2
View File
@@ -1,4 +1,3 @@
# Build artifacts
*.exe
elf2dol
vtable.lcf
__pycache__
tools/Makefile
-8
View File
@@ -1,8 +0,0 @@
CC := cc
CFLAGS := -O3 -Wall -s
elf2dol: elf2dol.c
$(CC) $(CFLAGS) -o $@ $^
clean:
$(RM) elf2dol
tools/conflict.py
+438
View File
@@ -0,0 +1,438 @@
"""
conflict.py - Finds conflicts between in main.dol that prevents it from matching.
"""
import sys
import logging
from pathlib import Path
from collections import defaultdict
try:
import click
from rich.logging import RichHandler
from rich.console import Console
except ImportError as e:
MISSING_PREREQUISITES = (
f"Missing prerequisite python module {e}.\n"
f"Run `python3 -m pip install --user -r tools/requirements.txt` to install prerequisites."
)
print(MISSING_PREREQUISITES, file=sys.stderr)
sys.exit(1)
class PathPath(click.Path):
def convert(self, value, param, ctx):
return Path(super().convert(value, param, ctx))
VERSION = "1.0"
CONSOLE = Console()
logging.basicConfig(
level="NOTSET",
format="%(message)s",
datefmt="[%X]",
handlers=[RichHandler(console=CONSOLE, rich_tracebacks=True)],
)
LOG = logging.getLogger("rich")
LOG.setLevel(logging.INFO)
@click.group()
@click.version_option(VERSION)
def conflict():
"""Finds conflicts between in main.dol that prevents it from matching."""
pass
class ConflictException(Exception):
pass
def try_hex(value, padding):
if value == None:
return value
if not isinstance(value, int):
return value
return "0x{0:0{1}X}".format(value, padding)
def normalize_name(name):
if name == None:
return None
# literals will have different indices, thus we cannot rely on their name
if name.startswith("@") or name.startswith("lit_"):
return None
return name
def is_literal(name):
return name.startswith("@") or name.startswith("lit_")
def name_match(A, B, addr):
if A == B:
return True
elif is_literal(A) and is_literal(B):
return True
elif A == B.replace("_o_iconv_cpp", "_cpp"): # TODO: remove, not needed any more
return True
elif A == f"func_{addr:08X}":
return True
elif A == f"data_{addr:08X}":
return True
elif B == f"func_{addr:08X}":
return True
elif B == f"data_{addr:08X}":
return True
return False
#
# All
#
@conflict.command(name="all")
@click.option(
"--build_path",
"build_path",
required=False,
type=PathPath(file_okay=False, dir_okay=True),
default="build/dolzel2/",
)
@click.option(
"--expected_path",
"expected_path",
required=False,
type=PathPath(file_okay=False, dir_okay=True),
default="expected/build/dolzel2/",
)
def conflict_all(build_path, expected_path):
"""Run all conflict checks."""
try:
sections(build_path, expected_path)
except ConflictException as exception:
LOG.error(exception)
try:
symbols(build_path, expected_path)
except ConflictException as exception:
LOG.error(exception)
CONSOLE.print("no conflicts were found 😊")
#
# Sections
#
@conflict.command(name="sections")
@click.option(
"--build_path",
"build_path",
required=False,
type=PathPath(file_okay=False, dir_okay=True),
default="build/dolzel2/",
)
@click.option(
"--expected_path",
"expected_path",
required=False,
type=PathPath(file_okay=False, dir_okay=True),
default="expected/build/dolzel2/",
)
def conflict_sections(build_path, expected_path):
"""Check if there are problems with the sections in the build compared with the expected build."""
try:
sections(build_path, expected_path)
except ConflictException as exception:
LOG.error(exception)
def sections(build_path, expected_path):
import libelf
import libdol
belf_file = build_path.joinpath("main.elf")
eelf_file = expected_path.joinpath("main.elf")
# load elf
build = libelf.load_object_from_path(
belf_file, skip_symbols=True, skip_relocations=True
)
expected = libelf.load_object_from_path(
eelf_file, skip_symbols=True, skip_relocations=True
)
SECTION_NAMES = [y for x, y in libdol.NAMES_FOR_INDEX.items()]
bsection_names = [k for k in build.sections if k in SECTION_NAMES]
esection_names = [k for k in expected.sections if k in SECTION_NAMES]
if len(bsection_names) != len(esection_names):
raise ConflictException(
f"number of elf sections does not match (expected: {len(esection_names)}, got: {len(bsection_names)})"
)
for bsection_name, esection_name in zip(bsection_names, esection_names):
if bsection_name != esection_name:
raise ConflictException(
f"section names does not match (expected: '{esection_name}', got: '{bsection_name}')"
)
bsection = build.sections[bsection_name]
esection = expected.sections[esection_name]
if type(bsection) != type(esection):
raise ConflictException(
f"'{bsection_name}' section kinds does not match (expected: '{type(esection)}', got: '{type(bsection)}')"
)
if bsection.addr != esection.addr:
raise ConflictException(
f"'{bsection_name}' section addresses does not match (expected: {try_hex(esection.addr,8)}, got: {try_hex(bsection.addr,8)})"
)
if bsection.size != esection.size:
info = []
info.append(
f"'{bsection_name}' section sizes does not match (expected: {try_hex(esection.size,6)}, got: {try_hex(bsection.size,6)})"
)
if bsection.header.sh_addr != 0:
info.append(f"build section:")
info.append(f" begin: 0x{bsection.header.sh_addr:08X}")
info.append(
f" end: 0x{bsection.header.sh_addr + bsection.size:08X}"
)
if esection.header.sh_addr != 0:
info.append(f"expected section:")
info.append(f" begin: 0x{esection.header.sh_addr:08X}")
info.append(
f" end: 0x{esection.header.sh_addr + esection.size:08X}"
)
raise ConflictException("\n".join(info))
for bsection_name, esection_name in zip(bsection_names, esection_names):
bsection = build.sections[bsection_name]
esection = expected.sections[esection_name]
if bsection.data != esection.data:
position = -1
for index, tup in enumerate(zip(esection.data, bsection.data)):
if tup[0] != tup[1]:
position = index
break
info = []
if position >= 0:
info.append(f"'{bsection_name}' sections data does not match")
info.append(
f"first difference is at position {position} (0x{position:04X}) (expected: 0x{tup[0]:02X}, got: 0x{tup[1]:02X})"
)
if bsection.header.sh_addr != 0:
build_location = bsection.header.sh_addr + position
info.append(f"build location:")
info.append(f" addr: 0x{build_location:08X}")
if esection.header.sh_addr != 0:
expected_location = esection.header.sh_addr + position
info.append(f"expected location:")
info.append(f" addr: 0x{expected_location:08X}")
else:
info.append(f"could not determine the byte difference")
raise ConflictException("\n".join(info))
# TODO: more checks?
#
# symbols
#
@conflict.command(name="symbols")
@click.option(
"--build_path",
"build_path",
required=False,
type=PathPath(file_okay=False, dir_okay=True),
default="build/dolzel2/",
)
@click.option(
"--expected_path",
"expected_path",
required=False,
type=PathPath(file_okay=False, dir_okay=True),
default="expected/build/dolzel2/",
)
def conflict_symbols(build_path, expected_path):
"""Check if there are problems with the symbols in the build compared with the expected build."""
try:
symbols(build_path, expected_path)
except ConflictException as exception:
LOG.error(exception)
def symbols(build_path, expected_path):
import libelf
import libdol
belf_file = build_path.joinpath("main.elf")
eelf_file = expected_path.joinpath("main.elf")
# load elf
build = libelf.load_object_from_path(
belf_file, skip_symbols=False, skip_relocations=True
)
expected = libelf.load_object_from_path(
eelf_file, skip_symbols=False, skip_relocations=True
)
# assign section address
for _, section in build.sections.items():
if section.header.sh_addr == 0:
continue
section.addr = section.header.sh_addr
for _, section in expected.sections.items():
if section.header.sh_addr == 0:
continue
section.addr = section.header.sh_addr
# build dictionary of symbol
def strip_filter(symbol):
if isinstance(symbol, libelf.AbsoluteSymbol):
# we're not checking for conflict between absolute symbols,
# they are generated by the lcf.py script and are only temporary.
return False
if symbol.name == None:
# we only care about symbols with names
return False
return True
build_stripped_symbols = [x for x in build.symbols if strip_filter(x)]
expected_stripped_symbols = [x for x in expected.symbols if strip_filter(x)]
build_name2symbols = defaultdict(list)
for symbol in build_stripped_symbols:
build_name2symbols[symbol.name].append(symbol)
expected_name2symbols = defaultdict(list)
for symbol in expected_stripped_symbols:
expected_name2symbols[symbol.name].append(symbol)
build_addr2sym = {k.offset: k for k in build_stripped_symbols}
expected_addr2sym = {k.offset: k for k in expected_stripped_symbols}
build_symbol_address_list = list(build_addr2sym.keys())
build_symbol_address_list.sort()
check_address_set = set()
for i, symbol_addr in enumerate(build_symbol_address_list):
symbol = build_addr2sym[symbol_addr]
if not symbol.offset in expected_addr2sym:
info = []
info.append(f"symbol not found")
info.append(f" section: {symbol.getSection().name}")
info.append(f" addr: 0x{symbol.offset:08X}")
info.append(f" size: 0x{symbol.size:05X}")
info.append(f" name: {symbol.name}")
raise ConflictException("\n".join(info))
expected_symbol = expected_addr2sym[symbol.offset]
if symbol.size != expected_symbol.size:
# because of dol2asm all data elements, before they are decompiled, will include
# padding. when decompiling the padding may get removed, and thus this tool will
# report a false-positive size difference. to fix this, find the offset to the next
# symbol (in the same section) and make sure it is located at the expected location.
next_symbol = None
current_section = symbol.getSection()
i += 1 # skip current symbol
if i < len(build_symbol_address_list):
i_addr = build_symbol_address_list[i]
i_symbol = build_addr2sym[i_addr]
if i_symbol.getSection() == current_section:
next_symbol = i_symbol
false_positive = False
if next_symbol:
difference = next_symbol.offset - symbol.offset
if difference == expected_symbol.size:
false_positive = True
if not false_positive:
info = []
info.append(
f"size difference (expected: 0x{expected_symbol.size:05X}, got: 0x{symbol.size:05X})"
)
info.append(f"symbol:")
info.append(f" section: {symbol.getSection().name}")
info.append(f" addr: 0x{symbol.offset:08X}")
info.append(f" size: 0x{symbol.size:05X}")
info.append(f" name: {symbol.name}")
info.append(f"expected symbol:")
info.append(f" section: {expected_symbol.getSection().name}")
info.append(f" addr: 0x{expected_symbol.offset:08X}")
info.append(f" size: 0x{expected_symbol.size:05X}")
info.append(f" name: {expected_symbol.name}")
raise ConflictException("\n".join(info))
if not name_match(symbol.name, expected_symbol.name, symbol.offset):
info = []
info.append(
f"name difference (expected: '{expected_symbol.name}', got: '{symbol.name}')"
)
info.append(f"symbol:")
info.append(f" section: {symbol.getSection().name}")
info.append(f" addr: 0x{symbol.offset:08X}")
info.append(f" size: 0x{symbol.size:05X}")
info.append(f" name: {symbol.name}")
info.append(f"expected symbol:")
info.append(f" section: {expected_symbol.getSection().name}")
info.append(f" addr: 0x{expected_symbol.offset:08X}")
info.append(f" size: 0x{expected_symbol.size:05X}")
info.append(f" name: {expected_symbol.name}")
raise ConflictException("\n".join(info))
check_address_set.add(symbol.offset)
expected_symbol_address_list = list(expected_addr2sym.keys())
expected_symbol_address_list.sort()
for symbol_addr in expected_symbol_address_list:
if symbol_addr in check_address_set:
continue
expected_symbol = build_addr2sym[symbol_addr]
info = []
info.append(f"missing symbol")
info.append(f"expected symbol:")
info.append(f" section: {expected_symbol.getSection().name}")
info.append(f" addr: 0x{expected_symbol.offset:08X}")
info.append(f" size: 0x{expected_symbol.size:05X}")
info.append(f" name: {expected_symbol.name}")
raise ConflictException("\n".join(info))
#
#
#
if __name__ == "__main__":
conflict()
tools/difftools.py
-294
View File
@@ -1,294 +0,0 @@
"""
difftools.py - Tools for finding differences between binaries
TODO: NOT FINISHED!
"""
VERSION = "1.0"
import os
import sys
import click
from pathlib import Path
from collections import defaultdict
class PathPath(click.Path):
def convert(self, value, param, ctx):
return Path(super().convert(value, param, ctx))
def fail(name):
sys.exit(1)
import libelf
@click.group()
@click.version_option(VERSION)
def difftools():
""" Tools for finding differences. """
pass
@difftools.command(name="addr")
@click.option('--truth', '-t', default="SYMDEF", type=click.Choice(['SYMDEF', 'EXPECTED', 'S', 'E'], case_sensitive=False))
@click.option('--build_path', 'build_path', required=False, type=PathPath(file_okay=False, dir_okay=True), default="build/dolzel2/")
@click.option('--expected_path', 'expected_path', required=False, type=PathPath(file_okay=False, dir_okay=True), default="expected/build/dolzel2/")
def find_bad_symbol_addr(truth, build_path, expected_path):
""" Find symbols which have the incorrect address. """
build_symbols = []
build_elf = build_path.joinpath("main.elf")
if not build_elf.exists():
fail(f"file not found: elf file '{build_elf}'")
build_symbols.extend(symbols_from_elf(build_elf))
expected_symbols = []
if truth == "EXPECTED" or truth == "E":
if not expected_path:
fail(f"when 'truth={truth}' the input argument 'expected_path' must be provided")
expected_elf = expected_path.joinpath("main.elf")
if not expected_elf.exists():
fail(f"file not found: expected elf file '{expected_elf}'")
expected_symbols.extend(symbols_from_elf(expected_elf))
else:
assert False
# match symbols by names
names = defaultdict(list)
for symbol in expected_symbols:
names[symbol.name].append(symbol)
build_addr_map = dict()
for symbol in build_symbols:
build_addr_map[elf_symbol_addr(symbol)] = symbol
# find matching symbols
last_difference = 0
build_symbols.sort(key =lambda x: elf_symbol_addr(x))
for symbol in build_symbols:
if not symbol.name in names:
continue
difference, closest_symbol = closest_match(symbol, names[symbol.name])
if difference != 0:
build_addr = elf_symbol_addr(symbol)
closest_addr = elf_symbol_addr(closest_symbol)
print("symbol with address difference found:")
print(f"\tname: '{symbol.name}'")
print(f"\tsection: '{symbol.section.name}'")
print(f"\tpath: '{symbol.object_path}'")
print("")
print(f"\tcompiled addr: 0x{build_addr:08X}")
print(f"\texpected addr: 0x{closest_addr:08X}")
print("")
previous_symbol, previous_addr = symbol_from_end(build_symbols, build_addr)
expected_symbol = symbol_at_addr(expected_symbols, previous_addr)
if previous_symbol and expected_symbol:
print("this is the expected symbol before the problem symbol:")
previous_start = elf_symbol_addr(previous_symbol)
previous_end = previous_start + previous_symbol.size
print(f"\t{previous_start:08X} {previous_end:08X} {previous_symbol.size:04X} {previous_symbol.name} (compiled)")
expected_start = elf_symbol_addr(expected_symbol)
expected_end = expected_start + expected_symbol.size
print(f"\t{expected_start:08X} {expected_end:08X} {expected_symbol.size:04X} {expected_symbol.name} (expected)")
if previous_symbol.size != expected_symbol.size:
print("\t!!! the size of this symbol is incorrect !!!")
sys.exit()
if expected_end != previous_end:
print("\t!!! the size of this symbol is incorrect !!!")
sys.exit()
inbetween_symbol = symbol_at_addr(expected_symbols, expected_end)
if inbetween_symbol:
print("found extra symbol in expected:")
start = elf_symbol_addr(inbetween_symbol)
end = start + inbetween_symbol.size
print(f"\t{start:08X} {end:08X} {inbetween_symbol.size:04X} {inbetween_symbol.name}")
print("\t!!! the compiled version is missing this symbol !!!")
sys.exit()
if symbol.size != closest_symbol.size:
print("symbol with size difference found:")
print(f"\tname: '{symbol.name}'")
print(f"\tsection: '{symbol.section.name}'")
print(f"\tpath: '{symbol.object_path}'")
print("")
print(f"\tcompiled size: 0x{symbol.size:04X}")
print(f"\texpected size: 0x{closest_symbol.size:04X}")
sys.exit()
sys.exit()
for symbol in expected_symbols:
addr = elf_symbol_addr(symbol)
if not addr in build_addr_map:
print("compiled is missing symbol:")
print(f"\tname: '{symbol.name}'")
print(f"\tsection: '{symbol.section.name}'")
print(f"\tpath: '{symbol.object_path}'")
print(f"\taddr: 0x{addr:08X}")
print(f"\tsize: 0x{size:04X}")
sys.exit()
@difftools.command(name="info")
@click.option('--truth', '-t', default="SYMDEF", type=click.Choice(['SYMDEF', 'EXPECTED', 'S', 'E'], case_sensitive=False))
@click.option('--build_path', 'build_path', required=False, type=PathPath(file_okay=False, dir_okay=True), default="build/dolzel2/")
@click.option('--expected_path', 'expected_path', required=False, type=PathPath(file_okay=False, dir_okay=True), default="expected/build/dolzel2/")
@click.argument('names', nargs=-1)
def info_about_symbol(truth, build_path, expected_path, names):
""" Display information about symbols (both in the build and the truth). """
build_symbols = []
build_elf = build_path.joinpath("main.elf")
if not build_elf.exists():
fail(f"file not found: elf file '{build_elf}'")
build_symbols.extend(symbols_from_elf(build_elf))
expected_symbols = []
if truth == "EXPECTED" or truth == "E":
if not expected_path:
fail(f"when 'truth={truth}' the input argument 'expected_path' must be provided")
expected_elf = expected_path.joinpath("main.elf")
if not expected_elf.exists():
fail(f"file not found: expected elf file '{expected_elf}'")
expected_symbols.extend(symbols_from_elf(expected_elf))
else:
assert False
for name in names:
build_symbols = symbols_by_name(build_symbols, name)
expected_symbols = symbols_by_name(expected_symbols, name)
print(f"###### {name} ######")
print("-- Build --")
for s in build_symbols:
print(f"\t{elf_symbol_addr(s):08X} {s.size:04X} {s.name}")
print("-- Expected --")
for s in expected_symbols:
print(f"\t{elf_symbol_addr(s):08X} {s.size:04X} {s.name}")
def symbols_by_name(symbols, name):
return [ symbol for symbol in symbols if symbol.name == name ]
def symbol_at_addr(symbols, addr):
for symbol in symbols:
start = elf_symbol_addr(symbol)
if start == addr:
return symbol
return None
def symbol_from_end(symbols, end_addr):
for symbol in symbols:
start = elf_symbol_addr(symbol)
end = start + symbol.size
if end >= end_addr:
return symbol, start
return None
def closest_match(symbol, matches):
difference = 0x1000000
closest_symbol = None
build_addr = elf_symbol_addr(symbol)
for found_symbol in matches:
expected_addr = elf_symbol_addr(found_symbol)
diff = abs(build_addr - expected_addr)
if diff < difference:
closest_symbol = found_symbol
difference = diff
return difference, closest_symbol
def elf_symbol_addr(symbol):
addr = symbol.offset
if symbol.section and symbol.section.addr:
addr += symbol.section.addr
return addr
def symbols_from_object(obj):
# TODO: possible to read what translation unit each symbol is in
symbols = []
for sym in obj.symbols:
if sym.isSection():
continue
if sym.isFile():
continue
if not isinstance(sym, libelf.OffsetSymbol):
continue
setattr(sym, 'object_path', obj.path) # TODO: not the best...
symbols.append(sym)
return symbols
def symbols_from_elf(path):
with open(path, 'rb') as file:
obj = libelf.load_object_from_file(path, path.name, file)
return symbols_from_object(obj)
return []
@difftools.command(name="section")
@click.option('--truth', '-t', default="SYMDEF", type=click.Choice(['SYMDEF', 'EXPECTED', 'S', 'E'], case_sensitive=False))
@click.option('--build_path', 'build_path', required=False, type=PathPath(file_okay=False, dir_okay=True), default="build/dolzel2/")
@click.option('--expected_path', 'expected_path', required=False, type=PathPath(file_okay=False, dir_okay=True), default="expected/build/dolzel2/")
@click.argument('section', nargs=1)
def section_diff(truth, build_path, expected_path, section):
build_symbols = []
build_elf = build_path.joinpath("main.elf")
if not build_elf.exists():
fail(f"file not found: elf file '{build_elf}'")
build_symbols.extend(symbols_from_elf(build_elf))
expected_symbols = []
if truth == "EXPECTED" or truth == "E":
if not expected_path:
fail(f"when 'truth={truth}' the input argument 'expected_path' must be provided")
expected_elf = expected_path.joinpath("main.elf")
if not expected_elf.exists():
fail(f"file not found: expected elf file '{expected_elf}'")
expected_symbols.extend(symbols_from_elf(expected_elf))
else:
assert False
build_symbols.sort(key=lambda x:elf_symbol_addr(x))
expected_symbols.sort(key=lambda x:elf_symbol_addr(x))
build_dict = dict()
expected_dict = dict()
for symbol in build_symbols:
if symbol.section.name != section:
continue
build_dict[elf_symbol_addr(symbol)] = symbol
for symbol in expected_symbols:
if symbol.section.name != section:
continue
expected_dict[elf_symbol_addr(symbol)] = symbol
keys = set([*build_dict.keys(), *expected_dict.keys()])
keys_list = list(keys)
keys_list.sort()
for key in keys_list:
in_build = key in build_dict
in_expected = key in expected_dict
if in_build and not in_expected:
print(f"+ {key:08X} {build_dict[key].size:04X} {build_dict[key].section.name:<10} '{build_dict[key].name}'")
elif not in_build and in_expected:
print(f"- {key:08X} {expected_dict[key].size:04X} {expected_dict[key].section.name:<10} '{expected_dict[key].name}'")
else:
build_sym = build_dict[key]
expected_sym = expected_dict[key]
print(f"= {key:08X} {build_sym.size:04X}/{expected_sym.size:04X} {build_sym.section.name:<10} '{build_sym.name}' '{expected_sym.name}'")
if __name__ == "__main__":
difftools()
tools/dol2asm.py
+113 -21
View File
@@ -5,33 +5,125 @@ dol2asm.py - Script for splitting .dol and .rel binaries into C++ and .s code.
This script only calls the underlaying libdol2asm code that does the heavy-lifting.
"""
import click
import libdol2asm
import sys
from pathlib import Path
try:
import click
import libdol2asm
except ImportError as e:
MISSING_PREREQUISITES = (
f"Missing prerequisite python module {e}.\n"
f"Run `python3 -m pip install --user -r tools/requirements.txt` to install prerequisites."
)
print(MISSING_PREREQUISITES, file=sys.stderr)
sys.exit(1)
@click.command()
@click.version_option(libdol2asm.VERSION)
@click.option('--debug/--no-debug', help="enable/disable debug logging", default=False)
@click.option('--game', 'game_path', help=f"Path to extracted game files. (same directory as 'main.dol' is in)", required=False, type=libdol2asm.util.PathPath(file_okay=False, dir_okay=True), default="game/")
@click.option('--lib-path', 'lib_path', help="Where to put generated library source files.", required=False, type=libdol2asm.util.PathPath(file_okay=False, dir_okay=True), default="libs/")
@click.option('--src-path', 'src_path', help="Where to put generated source files.", required=False, type=libdol2asm.util.PathPath(file_okay=False, dir_okay=True), default="src/")
@click.option('--asm-path', 'asm_path', help="Where to put generated asm files.", required=False, type=libdol2asm.util.PathPath(file_okay=False, dir_okay=True), default="asm/")
@click.option('--rel-path', 'rel_path', help="Where to put generated rel source files.", required=False, type=libdol2asm.util.PathPath(file_okay=False, dir_okay=True), default="rel/")
@click.option('--include-path', 'inc_path', help="Where to put generated include files.", required=False, type=libdol2asm.util.PathPath(file_okay=False, dir_okay=True), default="include/")
@click.option('--cpp/--no-cpp', 'cpp_gen', default=True)
@click.option('--asm/--no-asm', 'asm_gen', default=True)
@click.option('--makefile/--no-makefile', 'mk_gen', default=True)
@click.option('--symbols/--no-symbols', 'sym_gen', default=True)
@click.option('--rels/--no-rels', 'rel_gen', default=True)
@click.option('--threads', '-j', 'process_count', default=8)
@click.option('--select-module', '-g', 'select_modules', help="Select what modules to generate. Default is everything.", multiple=True)
@click.option('--select-asm', '-n', 'select_asm', multiple=True)
@click.option('--select-tu', '-t', 'select_tu', multiple=True)
def main(debug, game_path, asm_path, lib_path, src_path, rel_path, inc_path, mk_gen, cpp_gen, asm_gen, sym_gen, rel_gen, process_count, select_modules, select_tu, select_asm):
return libdol2asm.split(debug, game_path, lib_path, src_path, asm_path, rel_path, inc_path, mk_gen, cpp_gen, asm_gen, sym_gen, rel_gen, process_count, select_modules, select_tu, select_asm)
@click.option("--debug/--no-debug", help="enable/disable debug logging", default=False)
@click.option(
"--game",
"game_path",
help=f"Path to extracted game files. (same directory as 'main.dol' is in)",
required=False,
type=libdol2asm.util.PathPath(file_okay=False, dir_okay=True),
default="game/",
)
@click.option(
"--lib-path",
"lib_path",
help="Where to put generated library source files.",
required=False,
type=libdol2asm.util.PathPath(file_okay=False, dir_okay=True),
default="libs/",
)
@click.option(
"--src-path",
"src_path",
help="Where to put generated source files.",
required=False,
type=libdol2asm.util.PathPath(file_okay=False, dir_okay=True),
default="src/",
)
@click.option(
"--asm-path",
"asm_path",
help="Where to put generated asm files.",
required=False,
type=libdol2asm.util.PathPath(file_okay=False, dir_okay=True),
default="asm/",
)
@click.option(
"--rel-path",
"rel_path",
help="Where to put generated rel source files.",
required=False,
type=libdol2asm.util.PathPath(file_okay=False, dir_okay=True),
default="rel/",
)
@click.option(
"--include-path",
"inc_path",
help="Where to put generated include files.",
required=False,
type=libdol2asm.util.PathPath(file_okay=False, dir_okay=True),
default="include/",
)
@click.option("--cpp/--no-cpp", "cpp_gen", default=True)
@click.option("--asm/--no-asm", "asm_gen", default=True)
@click.option("--makefile/--no-makefile", "mk_gen", default=True)
@click.option("--symbols/--no-symbols", "sym_gen", default=True)
@click.option("--rels/--no-rels", "rel_gen", default=True)
@click.option("--threads", "-j", "process_count", default=8)
@click.option(
"--select-module",
"-g",
"select_modules",
help="Select what modules to generate. Default is everything.",
multiple=True,
)
@click.option("--select-asm", "-n", "select_asm", multiple=True)
@click.option("--select-tu", "-t", "select_tu", multiple=True)
def main(
debug,
game_path,
asm_path,
lib_path,
src_path,
rel_path,
inc_path,
mk_gen,
cpp_gen,
asm_gen,
sym_gen,
rel_gen,
process_count,
select_modules,
select_tu,
select_asm,
):
return libdol2asm.split(
debug,
game_path,
lib_path,
src_path,
asm_path,
rel_path,
inc_path,
mk_gen,
cpp_gen,
asm_gen,
sym_gen,
rel_gen,
process_count,
select_modules,
select_tu,
select_asm,
)
if __name__ == "__main__":
tools/dol2asm_settings.py
+1638
View File
File diff suppressed because it is too large Load Diff
tools/elf2dol/Makefile
+7
View File
@@ -0,0 +1,7 @@
ELF2DOL_CC := cc
ELF2DOL_CFLAGS := -O3 -Wall -s
$(ELF2DOL): include tools/elf2dol/Makefile
@echo [tools] building elf2dol
@$(ELF2DOL_CC) $(ELF2DOL_CFLAGS) -o $(ELF2DOL) tools/elf2dol/elf2dol.c
tools/elf2dol.c → tools/elf2dol/elf2dol.c
View File
tools/extract_game_assets.py
+76 -29
View File
@@ -12,78 +12,115 @@ numFileEntries = 0
"""
Returns the offset address and size of fst.bin
"""
def getFstInfo(handler, fstOffsetPosition):
fstOffset = int.from_bytes(bytearray(handler.read(4)), byteorder='big')
handler.seek(fstOffsetPosition + 4) # Get the size which is 4 bytes after the offset
fstSize = int.from_bytes(bytearray(handler.read(4)), byteorder='big')
fstOffset = int.from_bytes(bytearray(handler.read(4)), byteorder="big")
handler.seek(
fstOffsetPosition + 4
) # Get the size which is 4 bytes after the offset
fstSize = int.from_bytes(bytearray(handler.read(4)), byteorder="big")
return fstOffset, fstSize
"""
Parses the fst.bin into a list of dictionaries containing
the file entry type, the file/folder name, the ISO file offset/parent file entry, the file size/last file entry
"""
def parseFstBin(fstBinBytes):
currentByte = 0
numFileEntries = int.from_bytes(fstBinBytes[10:12], byteorder='big') # fst.bin offset
numFileEntries = int.from_bytes(
fstBinBytes[10:12], byteorder="big"
) # fst.bin offset
stringTableOffset = numFileEntries * 0xC
ret = []
while currentByte != (numFileEntries*12):
while currentByte != (numFileEntries * 12):
currentByte += 12
# lazy
if currentByte == (numFileEntries*12):
if currentByte == (numFileEntries * 12):
break
fileFolder = fstBinBytes[currentByte]
filenameOffset = int.from_bytes(fstBinBytes[currentByte+1:currentByte+4], byteorder='big')
fileOffsetOrParentEntryNum = int.from_bytes(fstBinBytes[currentByte+4:currentByte+8], byteorder='big')
fileSizeOrLastEntryNum = int.from_bytes(fstBinBytes[currentByte+8:currentByte+12], byteorder='big')
currentFilenameOffset = stringTableOffset+filenameOffset
filenameOffset = int.from_bytes(
fstBinBytes[currentByte + 1 : currentByte + 4], byteorder="big"
)
fileOffsetOrParentEntryNum = int.from_bytes(
fstBinBytes[currentByte + 4 : currentByte + 8], byteorder="big"
)
fileSizeOrLastEntryNum = int.from_bytes(
fstBinBytes[currentByte + 8 : currentByte + 12], byteorder="big"
)
currentFilenameOffset = stringTableOffset + filenameOffset
# Figure out the filename by checking for null string terminator
i = 0
while fstBinBytes[currentFilenameOffset+i] != 0:
while fstBinBytes[currentFilenameOffset + i] != 0:
i += 1
fileName = (fstBinBytes[currentFilenameOffset:currentFilenameOffset+i]).decode()
fileName = (
fstBinBytes[currentFilenameOffset : currentFilenameOffset + i]
).decode()
if fileFolder == 0:
ret.append({"type": "File","fileName": fileName,"fileOffset":fileOffsetOrParentEntryNum,"fileSize":fileSizeOrLastEntryNum})
ret.append(
{
"type": "File",
"fileName": fileName,
"fileOffset": fileOffsetOrParentEntryNum,
"fileSize": fileSizeOrLastEntryNum,
}
)
else:
ret.append({"type": "Folder","folderName": fileName,"parentFolderEntryNumber": fileOffsetOrParentEntryNum, "lastEntryNumber": fileSizeOrLastEntryNum})
ret.append(
{
"type": "Folder",
"folderName": fileName,
"parentFolderEntryNumber": fileOffsetOrParentEntryNum,
"lastEntryNumber": fileSizeOrLastEntryNum,
}
)
return ret
"""
Write the current folder to disk and return it's name/last entry number
"""
def writeFolder(parsedFstBin,i):
folderPath = i["folderName"]+"/"
def writeFolder(parsedFstBin, i):
folderPath = i["folderName"] + "/"
lastEntryNumber = i["lastEntryNumber"]
if i["parentFolderEntryNumber"] == 0:
if not os.path.exists(folderPath):
os.makedirs(folderPath)
else:
parentFolderEntry = parsedFstBin[i["parentFolderEntryNumber"]-1]
parentFolderEntry = parsedFstBin[i["parentFolderEntryNumber"] - 1]
while True:
folderPath = parentFolderEntry["folderName"] + "/" + folderPath
if parentFolderEntry["parentFolderEntryNumber"] == 0:
break
nextParentFolderEntryNumber = parentFolderEntry["parentFolderEntryNumber"]
parentFolderEntry = parsedFstBin[nextParentFolderEntryNumber-1]
parentFolderEntry = parsedFstBin[nextParentFolderEntryNumber - 1]
if not os.path.exists(folderPath):
os.makedirs(folderPath)
return folderPath, lastEntryNumber
"""
Use the parsed fst.bin contents to write assets to file
"""
def writeAssets(parsedFstBin, handler):
# Write the folder structure and files to disc
j = 0
@@ -92,21 +129,26 @@ def writeAssets(parsedFstBin, handler):
for i in parsedFstBin:
j += 1
if i["type"] == "Folder":
currentFolder, lastEntryNumber = writeFolder(parsedFstBin,i)
folderStack.append({"folderName": currentFolder, "lastEntryNumber": lastEntryNumber})
currentFolder, lastEntryNumber = writeFolder(parsedFstBin, i)
folderStack.append(
{"folderName": currentFolder, "lastEntryNumber": lastEntryNumber}
)
else:
handler.seek(i["fileOffset"])
with open((folderStack[-1]["folderName"]+i["fileName"]), "wb") as currentFile:
with open(
(folderStack[-1]["folderName"] + i["fileName"]), "wb"
) as currentFile:
currentFile.write(bytearray(handler.read(i["fileSize"])))
while folderStack[-1]["lastEntryNumber"] == j+1:
while folderStack[-1]["lastEntryNumber"] == j + 1:
folderStack.pop()
def main():
with open(sys.argv[1], "rb") as f:
def extract(path):
with open(path, "rb") as f:
# Seek to fst offset information and retrieve it
f.seek(fstInfoPosition)
fstOffset,fstSize = getFstInfo(f,fstInfoPosition)
fstOffset, fstSize = getFstInfo(f, fstInfoPosition)
# Seek to fst.bin and retrieve it
f.seek(fstOffset)
@@ -118,5 +160,10 @@ def main():
# Write assets to file
writeAssets(parsedFstBin, f)
def main():
extract(sys.argv[1], "rb")
if __name__ == "__main__":
main()
main()
tools/find_unused_asm.py
-32
View File
@@ -1,32 +0,0 @@
"""
Use as `python tools/find_unused_asm.py | xargs rm`
"""
import inotify.adapters
from inotify.constants import IN_OPEN
from pathlib import Path
import subprocess
from sys import stderr
asm_files = set(Path('include/').glob('**/*.s'))
stderr.write('==> clean\n')
subprocess.run(['make', 'clean'], stdout=subprocess.DEVNULL)
stderr.write('==> set up watches\n')
ino = inotify.adapters.Inotify()
for p in asm_files:
ino.add_watch(str(p), mask=IN_OPEN)
stderr.write('==> run make\n')
subprocess.run(['make', '-j'], stdout=subprocess.DEVNULL)
opened_paths = set()
for evt in ino.event_gen(timeout_s=1):
if evt:
(header, type_names, path, filename) = evt
opened_paths.add(Path(path))
unused_asm = asm_files - opened_paths
for p in unused_asm:
print(str(p))
tools/lcf.py
+76 -58
View File
@@ -1,31 +1,40 @@
"""
lcf.py
Generates the .lcf file used for the linker. This will auto force actives missing functions and data
and apply some fixes with makes it easier to decompile.
and apply some fixes which makes it easier to decompile.
"""
import importlib
import click
from libdol2asm import settings
import libelf
import libar
from pathlib import Path
import io
import sys
import os
from pathlib import Path
try:
import click
import libelf
import libar
import dol2asm_settings
except ImportError as e:
MISSING_PREREQUISITES = (
f"Missing prerequisite python module {e}.\n"
f"Run `python3 -m pip install --user -r tools/requirements.txt` to install prerequisites."
)
print(MISSING_PREREQUISITES, file=sys.stderr)
sys.exit(1)
VERSION = "1.0"
# laod the symbol definition file for main.dol
sys.path.append('defs')
# load the symbol definition file for main.dol
sys.path.append("defs")
def lcf_generate(output_path):
""" Script for generating .lcf files """
"""Script for generating .lcf files"""
import module0
@@ -36,11 +45,11 @@ def lcf_generate(output_path):
# load object files from the 'build/o_files', this way we need no list of
# object files in the python code.
with open("build/o_files", 'r') as content_file:
with open("build/o_files", "r") as content_file:
o_files = content_file.read().strip().split(" ")
for o_file in o_files:
with open(o_file, 'rb') as file:
with open(o_file, "rb") as file:
obj = libelf.load_object_from_file(None, o_file, file)
symbols.extend(get_symbols_from_object_file(obj))
@@ -62,7 +71,8 @@ def lcf_generate(output_path):
file.write("\t} > text\n")
file.write(
"\t_stack_addr = (_f_sbss2 + SIZEOF(.sbss2) + 65536 + 0x7) & ~0x7;\n")
"\t_stack_addr = (_f_sbss2 + SIZEOF(.sbss2) + 65536 + 0x7) & ~0x7;\n"
)
file.write("\t_stack_end = _f_sbss2 + SIZEOF(.sbss2);\n")
file.write("\t_db_stack_addr = (_stack_addr + 0x2000);\n")
file.write("\t_db_stack_end = _stack_addr;\n")
@@ -80,9 +90,9 @@ def lcf_generate(output_path):
names = base_names - main_names
for name in names:
symbol = module0.SYMBOLS[module0.SYMBOL_NAMES[name]]
if symbol['type'] == "StringBase": # @stringBase0 is handled below
if symbol["type"] == "StringBase": # @stringBase0 is handled below
continue
if symbol['type'] == "LinkerGenerated": # linker handles these symbols
if symbol["type"] == "LinkerGenerated": # linker handles these symbols
continue
file.write(f"\t\"{symbol['label']}\" = 0x{symbol['addr']:08X};\n")
@@ -95,40 +105,40 @@ def lcf_generate(output_path):
# that the @stringBase0 symbol is never used and strip it.
file.write("\t/* @stringBase0 */\n")
for x in module0.SYMBOLS:
if x['type'] == "StringBase":
file.write("\t\"%s\" = 0x%08X;\n" % (x['label'], x['addr']))
if x["type"] == "StringBase":
file.write('\t"%s" = 0x%08X;\n' % (x["label"], x["addr"]))
file.write("}\n")
file.write("\n")
file.write("FORCEACTIVE {\n")
for f in FORCE_ACTIVE:
file.write("\t\"%s\"\n" % f)
file.write('\t"%s"\n' % f)
file.write("\n")
file.write("\t/* unreferenced symbols */\n")
for x in module0.SYMBOLS:
k = x['label']
if x['type'] == "StringBase":
k = x["label"]
if x["type"] == "StringBase":
continue
require_force_active = False
# if the symbol is not reachable from the __start add it as forceactive
if not x['is_reachable'] or sum(x['r']) == 0:
if not x["is_reachable"] or sum(x["r"]) == 0:
require_force_active = True
if require_force_active:
file.write(f"\t\"{x['label']}\"\n")
if not x['label'] in main_names:
if not x["label"] in main_names:
file.write(f"\t\"{x['name']}\"\n")
for x in module0.SYMBOLS:
if x['type'] == "StringBase":
if x["type"] == "StringBase":
continue
if x['is_reachable']:
if x['label'] != x['name']:
if x["is_reachable"]:
if x["label"] != x["name"]:
file.write(f"\t\"{x['name']}\"\n")
for symbol in symbols:
@@ -136,7 +146,7 @@ def lcf_generate(output_path):
continue
if "__template" in symbol.name:
file.write("\t\"%s\"\n" % (symbol.name))
file.write('\t"%s"\n' % (symbol.name))
file.write("\n")
file.write("}\n")
@@ -146,27 +156,20 @@ def lcf_generate(output_path):
def rel_lcf_generate(module_index, output_path):
module = importlib.import_module(f"module{module_index}")
base = settings.REL_TEMP_LOCATION[module.LIBRARIES[0].split(
"/")[-1] + ".rel"]
base = dol2asm_settings.REL_TEMP_LOCATION[
module.LIBRARIES[0].split("/")[-1] + ".rel"
]
# load object files from the 'build/o_files', this way we need no list of
# object files in the python code.
with open(f"build/M{module_index}_ofiles", 'r') as content_file:
with open(f"build/M{module_index}_ofiles", "r") as content_file:
all_files = content_file.read().strip().split(" ")
path = f"build/dolzel2/rel/{module.LIBRARIES[0]}"
archives = [
path
for path in all_files
if path.endswith(".a")
]
archives = [path for path in all_files if path.endswith(".a")]
o_files = [
path
for path in all_files
if path.endswith(".o")
]
o_files = [path for path in all_files if path.endswith(".o")]
# load symbols from compiled files
symbols = []
@@ -174,7 +177,7 @@ def rel_lcf_generate(module_index, output_path):
symbols.extend(load_archive(archive))
for o_file in o_files:
with open(o_file, 'rb') as file:
with open(o_file, "rb") as file:
obj = libelf.load_object_from_file(None, o_file, file)
symbols.extend(get_symbols_from_object_file(obj))
@@ -186,7 +189,7 @@ def rel_lcf_generate(module_index, output_path):
for name, align in REL_SECTIONS:
file.write(f"\t\t{name} :{{}}\n")
#file.write("\t\t/DISCARD/ : { *(.dead) }\n")
# file.write("\t\t/DISCARD/ : { *(.dead) }\n")
file.write("\t}\n")
@@ -202,13 +205,14 @@ def rel_lcf_generate(module_index, output_path):
names = base_names - main_names
for name in names:
symbol = module.SYMBOLS[module.SYMBOL_NAMES[name]]
if symbol['type'] == "StringBase": # @stringBase0 is handled below
if symbol["type"] == "StringBase": # @stringBase0 is handled below
continue
if symbol['type'] == "LinkerGenerated": # linker handles these symbols
if symbol["type"] == "LinkerGenerated": # linker handles these symbols
continue
file.write(
f"\t\"{symbol['label']}\" = __rel_base + 0x{symbol['addr'] - base:08X}; /* 0x{symbol['addr']:08X} */\n")
f"\t\"{symbol['label']}\" = __rel_base + 0x{symbol['addr'] - base:08X}; /* 0x{symbol['addr']:08X} */\n"
)
file.write("\n")
file.write("}\n")
@@ -222,27 +226,27 @@ def rel_lcf_generate(module_index, output_path):
file.write("\t/* unreferenced symbols */\n")
for x in module.SYMBOLS:
k = x['label']
if x['type'] == "StringBase":
k = x["label"]
if x["type"] == "StringBase":
continue
require_force_active = False
# if the symbol is not reachable from the __start add it as forceactive
if not x['is_reachable'] and not x['static']:
if not x["is_reachable"] and not x["static"]:
require_force_active = True
if require_force_active:
file.write(f"\t\"{x['label']}\"\n")
if not x['label'] in main_names:
if not x["label"] in main_names:
file.write(f"\t\"{x['name']}\"\n")
for x in module.SYMBOLS:
if x['type'] == "StringBase":
if x["type"] == "StringBase":
continue
if x['is_reachable']:
if x['label'] != x['name'] and x['name']:
if x["is_reachable"]:
if x["label"] != x["name"] and x["name"]:
file.write(f"\t\"{x['name']}\"\n")
for symbol in symbols:
@@ -250,7 +254,7 @@ def rel_lcf_generate(module_index, output_path):
continue
if "__template" in symbol.name:
file.write("\t\"%s\"\n" % (symbol.name))
file.write('\t"%s"\n' % (symbol.name))
file.write("\n")
file.write("}\n")
@@ -293,7 +297,7 @@ SECTIONS = [
(".sdata2", 0x20),
(".sbss2", 0x20),
(".stack", 0x100),
#(".dead", 0x100),
# (".dead", 0x100),
]
REL_SECTIONS = [
@@ -371,14 +375,28 @@ def lcf():
@lcf.command(name="dol")
@click.option('--output', '-o', 'output_path', required=False, type=PathPath(file_okay=True, dir_okay=False), default="build/dolzel2/ldscript.lcf")
@click.option(
"--output",
"-o",
"output_path",
required=False,
type=PathPath(file_okay=True, dir_okay=False),
default="build/dolzel2/ldscript.lcf",
)
def dol(output_path):
lcf_generate(output_path)
@lcf.command(name="rel")
@click.option('--output', '-o', 'output_path', required=False, type=PathPath(file_okay=True, dir_okay=False), default="build/dolzel2/ldscript.lcf")
@click.argument('module', metavar="<MODULE>", nargs=1)
@click.option(
"--output",
"-o",
"output_path",
required=False,
type=PathPath(file_okay=True, dir_okay=False),
default="build/dolzel2/ldscript.lcf",
)
@click.argument("module", metavar="<MODULE>", nargs=1)
def rel(output_path, module):
rel_lcf_generate(module, output_path)
tools/libdol2asm/__init__.py
+2 -6
View File
@@ -1,13 +1,9 @@
from pathlib import Path
from . import globals
from . import split_asm
from . import settings
VERSION = globals.VERSION
def split(debug_logging, game_path, lib_path, src_path, asm_path, rel_path, inc_path, mk_gen, cpp_gen, asm_gen, sym_gen, rel_gen, process_count, select_modules, select_tu, select_asm):
from . import split_asm
from . import settings
splitter = split_asm.Dol2AsmSplitter(debug_logging, game_path, lib_path, src_path, asm_path, rel_path, inc_path, mk_gen, cpp_gen, asm_gen, sym_gen, rel_gen, process_count, select_modules, select_tu, select_asm)
return splitter.main()
tools/libdol2asm/requirements.txt
+9
View File
@@ -0,0 +1,9 @@
rich
click
intervaltree
yaz0
numpy
capstone
aiofiles
GitPython
hexdump
tools/libelf/object.py
+85 -83
View File
@@ -33,7 +33,7 @@ class Object:
path: Path = None
executable: bool = False
def load_object_from_file(path, name, file) -> Object:
def load_object_from_file(path, name, file, skip_symbols = False, skip_relocations = False) -> Object:
obj = Object()
obj.path = path
obj.name = name
@@ -135,98 +135,100 @@ def load_object_from_file(path, name, file) -> Object:
if section.name:
obj.sections[section.name] = section
# Find all symbols
for symtab in obj.sym_sections.values():
if not symtab.header.sh_link in obj.str_sections:
raise ElfException("symbol table '%s' is not referenceing a valid string table section (sh_link: %i)" % (
symtab.name, symtab.header.sh_link))
if not skip_symbols:
# Find all symbols
for symtab in obj.sym_sections.values():
if not symtab.header.sh_link in obj.str_sections:
raise ElfException("symbol table '%s' is not referenceing a valid string table section (sh_link: %i)" % (
symtab.name, symtab.header.sh_link))
symtab.object_offset = len(obj.symbols)
strtab = obj.str_sections[symtab.header.sh_link]
for i,sym in enumerate(symtab.symbols):
if i == 0:
symbol = NullSymbol(sym)
symtab.object_offset = len(obj.symbols)
strtab = obj.str_sections[symtab.header.sh_link]
for i,sym in enumerate(symtab.symbols):
if i == 0:
symbol = NullSymbol(sym)
symbol.object = obj
obj.symbols.append(symbol)
continue
name = None
if sym.st_name:
name = strtab.readString(sym.st_name)
symbol = None
if sym.st_shndx == elf.SHN_UNDEF:
symbol = UndefSymbol(sym, name)
elif sym.st_shndx == elf.SHN_ABS:
symbol = AbsoluteSymbol(sym, name, sym.st_value)
else:
if not sym.st_shndx in idx_sections:
raise ElfException("symbol '%s' has invalid section-id (st_shndx: %i)" % (name, sym.st_shndx))
s = idx_sections[sym.st_shndx]
symbol = OffsetSymbol(sym, name, idx_sections[sym.st_shndx], sym.st_value)
assert symbol
symbol.object = obj
if symbol.name:
obj.symbol_map[symbol.name].append(symbol)
obj.symbols.append(symbol)
continue
name = None
if sym.st_name:
name = strtab.readString(sym.st_name)
symbol = None
if sym.st_shndx == elf.SHN_UNDEF:
symbol = UndefSymbol(sym, name)
elif sym.st_shndx == elf.SHN_ABS:
symbol = AbsoluteSymbol(sym, name, sym.st_value)
else:
if not sym.st_shndx in idx_sections:
raise ElfException("symbol '%s' has invalid section-id (st_shndx: %i)" % (name, sym.st_shndx))
s = idx_sections[sym.st_shndx]
symbol = OffsetSymbol(sym, name, idx_sections[sym.st_shndx], sym.st_value)
if not skip_relocations:
# Find all relocations
for rela_section in obj.rela_sections.values():
if not rela_section.header.sh_link in obj.sym_sections:
raise ElfException("relocation section '%s' is not referenceing a valid symbol table section (sh_link: %i)" % (
symtab.name, rela_section.header.sh_link))
if not rela_section.header.sh_info in idx_sections:
raise ElfException("relocation section '%s' is not referenceing a valid section (sh_info: %i)" % (
symtab.name, rela_section.header.sh_info))
assert symbol
symbol.object = obj
symtab = obj.sym_sections[rela_section.header.sh_link]
modify = idx_sections[rela_section.header.sh_info]
if symbol.name:
obj.symbol_map[symbol.name].append(symbol)
section_relocations = []
for rela in rela_section.relocations:
type = elf.R_TYPE(rela.r_info)
sym_id = elf.R_SYM(rela.r_info)
if not type in RELOCATION_NAMES:
raise ElfException("unsupported relocation type: 0x%02X (in '%s')" % (type, path))
obj.symbols.append(symbol)
if sym_id < 0 or sym_id >= len(symtab.symbols):
# report warning?
# main.elf will generate relocation sections with invalid symbol indices
# raise ElfException("invalid symbol index: %i (%i symbols)" % (sym_id, len(symtab.symbols)))
continue
symbol = obj.symbols[symtab.object_offset + sym_id]
# Find all relocations
for rela_section in obj.rela_sections.values():
if not rela_section.header.sh_link in obj.sym_sections:
raise ElfException("relocation section '%s' is not referenceing a valid symbol table section (sh_link: %i)" % (
symtab.name, rela_section.header.sh_link))
if not rela_section.header.sh_info in idx_sections:
raise ElfException("relocation section '%s' is not referenceing a valid section (sh_info: %i)" % (
symtab.name, rela_section.header.sh_info))
relocation = None
if type == 1:
relocation = R_PPC_ADDR32(type, symbol, modify, rela.r_offset, rela.r_addend)
elif type == 3:
relocation = R_PPC_ADDR16(type, symbol, modify, rela.r_offset, rela.r_addend)
elif type == 4:
relocation = R_PPC_ADDR16_LO(type, symbol, modify, rela.r_offset, rela.r_addend)
elif type == 5:
relocation = R_PPC_ADDR16_HI(type, symbol, modify, rela.r_offset, rela.r_addend)
elif type == 6:
relocation = R_PPC_ADDR16_HA(type, symbol, modify, rela.r_offset, rela.r_addend)
elif type == 10:
relocation = R_PPC_REL24(type, symbol, modify, rela.r_offset, rela.r_addend)
elif type == 11:
relocation = R_PPC_REL14(type, symbol, modify, rela.r_offset, rela.r_addend)
elif type == 109:
relocation = R_PPC_EMB_SDA21(type, symbol, modify, rela.r_offset, rela.r_addend)
else:
print("unsupported relocation type: 0x%02X \"%s\" (in '%s')" % (type, RELOCATION_NAMES[type], path), file = sys.stderr)
continue
assert relocation
section_relocations.append(relocation)
obj.relocations.append(relocation)
symtab = obj.sym_sections[rela_section.header.sh_link]
modify = idx_sections[rela_section.header.sh_info]
section_relocations = []
for rela in rela_section.relocations:
type = elf.R_TYPE(rela.r_info)
sym_id = elf.R_SYM(rela.r_info)
if not type in RELOCATION_NAMES:
raise ElfException("unsupported relocation type: 0x%02X (in '%s')" % (type, path))
if sym_id < 0 or sym_id >= len(symtab.symbols):
# report warning?
# main.elf will generate relocation sections with invalid symbol indices
# raise ElfException("invalid symbol index: %i (%i symbols)" % (sym_id, len(symtab.symbols)))
continue
symbol = obj.symbols[symtab.object_offset + sym_id]
relocation = None
if type == 1:
relocation = R_PPC_ADDR32(type, symbol, modify, rela.r_offset, rela.r_addend)
elif type == 3:
relocation = R_PPC_ADDR16(type, symbol, modify, rela.r_offset, rela.r_addend)
elif type == 4:
relocation = R_PPC_ADDR16_LO(type, symbol, modify, rela.r_offset, rela.r_addend)
elif type == 5:
relocation = R_PPC_ADDR16_HI(type, symbol, modify, rela.r_offset, rela.r_addend)
elif type == 6:
relocation = R_PPC_ADDR16_HA(type, symbol, modify, rela.r_offset, rela.r_addend)
elif type == 10:
relocation = R_PPC_REL24(type, symbol, modify, rela.r_offset, rela.r_addend)
elif type == 11:
relocation = R_PPC_REL14(type, symbol, modify, rela.r_offset, rela.r_addend)
elif type == 109:
relocation = R_PPC_EMB_SDA21(type, symbol, modify, rela.r_offset, rela.r_addend)
else:
print("unsupported relocation type: 0x%02X \"%s\" (in '%s')" % (type, RELOCATION_NAMES[type], path), file = sys.stderr)
continue
assert relocation
section_relocations.append(relocation)
obj.relocations.append(relocation)
obj.section_relocations.append((rela_section.name, section_relocations))
obj.section_relocations.append((rela_section.name, section_relocations))
return obj
def load_object_from_path(path) -> Object:
def load_object_from_path(path, skip_symbols = False, skip_relocations = False) -> Object:
with open(path, 'rb') as file:
return load_object_from_file(path, path.parts[-1], file)
return load_object_from_file(path, path.parts[-1], file, skip_symbols, skip_relocations)
tools/makerel.py
+120 -76
View File
@@ -4,23 +4,31 @@ makerel.py - Generate .rel files from .plf files and a static binary
"""
import click
import sys
import rich
import logging
import glob
import os
import libelf
import librel
import yaz0
import traceback
from pathlib import Path
from collections import defaultdict
from dataclasses import dataclass, field
from typing import List, Set, Tuple, Dict
from rich.logging import RichHandler
from rich.console import Console
try:
import libelf
import librel
import click
import logging
from rich.logging import RichHandler
from rich.console import Console
except ImportError as e:
MISSING_PREREQUISITES = (
f"Missing prerequisite python module {e}.\n"
f"Run `python3 -m pip install --user -r tools/requirements.txt` to install prerequisites."
)
print(MISSING_PREREQUISITES, file=sys.stderr)
sys.exit(1)
VERSION = "1.0"
CONSOLE = Console()
@@ -29,21 +37,13 @@ logging.basicConfig(
level="NOTSET",
format="%(message)s",
datefmt="[%X]",
handlers=[RichHandler(console=CONSOLE, rich_tracebacks=True)]
handlers=[RichHandler(console=CONSOLE, rich_tracebacks=True)],
)
LOG = logging.getLogger("rich")
LOG.setLevel(logging.INFO)
SECTION_MASK = {
".init",
".text",
".ctors",
".dtors",
".rodata",
".data",
".bss"
}
SECTION_MASK = {".init", ".text", ".ctors", ".dtors", ".rodata", ".data", ".bss"}
REL_SECTION_MASK = {
".rela.init",
@@ -55,17 +55,19 @@ REL_SECTION_MASK = {
".rela.bss",
}
@click.group()
@click.version_option(VERSION)
def makerel():
pass
@makerel.command(name="unresolved")
@click.option('--debug/--no-debug')
@click.option("--debug/--no-debug")
@click.option("--output", "-o", default="forceactive.txt", required=True)
@click.argument("str_paths", metavar='<ELFs>', nargs=-1)
@click.argument("str_paths", metavar="<ELFs>", nargs=-1)
def unresolved(debug, output, str_paths):
""" Generate a list of symbols which must be in the static executable (and other RELs). """
"""Generate a list of symbols which must be in the static executable (and other RELs)."""
if debug:
LOG.setLevel(logging.DEBUG)
@@ -73,8 +75,7 @@ def unresolved(debug, output, str_paths):
static, plfs = load_elfs(str_paths)
if static:
LOG.error(
f"unresolved does not handle executable files '{static.path}'")
LOG.error(f"unresolved does not handle executable files '{static.path}'")
sys.exit(1)
undef_symbols = set()
@@ -97,15 +98,21 @@ def unresolved(debug, output, str_paths):
@makerel.command(name="build")
@click.option('--debug/--no-debug')
@click.option('--yaz0', '-y', 'compress_yaz0')
@click.option("--id-offset", '-i', 'rel_id_offset', default=1)
@click.option("--spoof-path", '-q', 'spoof_path', default="D:\\zeldaGC_USA\\dolzel2\\bin\\Final\\")
@click.option("--string-table", '-s', 'string_path', required=True)
@click.option("--symbols", default="ELF", type=click.Choice(["ELF", "DEFS"], case_sensitive=False))
@click.argument("str_paths", metavar='<ELFs>', nargs=-1)
def build(debug, symbols, str_paths, rel_id_offset, compress_yaz0, spoof_path, string_path):
""" Build RELs files from a list of plfs files. """
@click.option("--debug/--no-debug")
@click.option("--yaz0", "-y", "compress_yaz0")
@click.option("--id-offset", "-i", "rel_id_offset", default=1)
@click.option(
"--spoof-path", "-q", "spoof_path", default="D:\\zeldaGC_USA\\dolzel2\\bin\\Final\\"
)
@click.option("--string-table", "-s", "string_path", required=True)
@click.option(
"--symbols", default="ELF", type=click.Choice(["ELF", "DEFS"], case_sensitive=False)
)
@click.argument("str_paths", metavar="<ELFs>", nargs=-1)
def build(
debug, symbols, str_paths, rel_id_offset, compress_yaz0, spoof_path, string_path
):
"""Build RELs files from a list of plfs files."""
if debug:
LOG.setLevel(logging.DEBUG)
@@ -115,7 +122,7 @@ def build(debug, symbols, str_paths, rel_id_offset, compress_yaz0, spoof_path, s
LOG.error(f"static executable ('main.elf') expected")
sys.exit(1)
#
#
id = rel_id_offset
elfs = []
for plf in plfs:
@@ -123,7 +130,7 @@ def build(debug, symbols, str_paths, rel_id_offset, compress_yaz0, spoof_path, s
rel = IndexedElf(0, plf)
else:
rel = IndexedElf(id, plf)
id += 1
id += 1
elfs.append(rel)
# sort relocations
@@ -132,7 +139,7 @@ def build(debug, symbols, str_paths, rel_id_offset, compress_yaz0, spoof_path, s
elf._unresolved = elf.plf.symbol_map["_unresolved"][0]
for _, relocations in elf.plf.section_relocations:
relocations.sort(key=lambda r: r.offset)
relocations.sort(key=lambda r: r.offset)
# symbol table
symbol_table = dict()
@@ -155,17 +162,30 @@ def build(debug, symbols, str_paths, rel_id_offset, compress_yaz0, spoof_path, s
string_list.write()
def apply_rel24_relocation(relocation, section, symbol):
if not symbol or not isinstance(symbol, libelf.OffsetSymbol):
return False
try:
if librel.apply_relocation(relocation.type, 0, section.data, 0, relocation.offset, symbol.offset, relocation.addend):
if librel.apply_relocation(
relocation.type,
0,
section.data,
0,
relocation.offset,
symbol.offset,
relocation.addend,
):
return True
except librel.RELRelocationException as e:
LOG.error(f"applying relocation failed!")
LOG.error(f"relocation: {librel.RELOCATION_NAMES[relocation.type]} {relocation.offset:04X}")
LOG.error(f"section: {section.header.sh_addr:08X} {section.header.sh_size:04X} {section.name}")
LOG.error(
f"relocation: {librel.RELOCATION_NAMES[relocation.type]} {relocation.offset:04X}"
)
LOG.error(
f"section: {section.header.sh_addr:08X} {section.header.sh_size:04X} {section.name}"
)
LOG.error(f"symbol: {symbol.offset:08X} {symbol.name}+0x{relocation.addend:X}")
LOG.error(e)
CONSOLE.print_exception()
@@ -175,6 +195,7 @@ def apply_rel24_relocation(relocation, section, symbol):
return False
@dataclass
class StringList:
output_path: str
@@ -189,9 +210,9 @@ class StringList:
self.data += name
return offset, len(name)
def write(self):
def write(self):
if len(self.data) > 0:
with open(self.output_path, 'w') as file:
with open(self.output_path, "w") as file:
file.write(self.data)
@@ -200,7 +221,7 @@ class ImpTable:
id: int
last_section: int
section_offset: int = 0
relocations: List[Tuple[int,int,int,int]] = field(default_factory=list)
relocations: List[Tuple[int, int, int, int]] = field(default_factory=list)
rel_offset: int = 0
def section(self, section_id):
@@ -224,7 +245,7 @@ class IndexedElf:
plf: libelf.Object
_unresolved: libelf.Symbol = None
imp_tables: Dict[int,ImpTable] = field(default_factory=dict)
imp_tables: Dict[int, ImpTable] = field(default_factory=dict)
imp_table_order: List[int] = field(default_factory=list)
complete_relocations: Set[libelf.Relocation] = field(default_factory=set)
@@ -250,8 +271,7 @@ class IndexedElf:
if replace_symbol:
my_symbol = replace_symbol
#LOG.info(f"relocation for: {my_symbol.name} ({type(my_symbol).__name__})")
# LOG.info(f"relocation for: {my_symbol.name} ({type(my_symbol).__name__})")
other = None
if isinstance(my_symbol, libelf.UndefSymbol):
if my_symbol.name in symbol_table:
@@ -291,14 +311,20 @@ class IndexedElf:
found_section = None
for section in other.plf.sections.values():
if ext_symbol.address >= section.header.sh_addr and ext_symbol.address < section.header.sh_addr + section.header.sh_size:
if (
ext_symbol.address >= section.header.sh_addr
and ext_symbol.address
< section.header.sh_addr + section.header.sh_size
):
found_section = section
break
if found_section:
ext_symbol.section = found_section
else:
LOG.error(f"error no-section provided for relocation of: {my_symbol.name} ({type(my_symbol).__name__}) ({self.plf.name} <- {other.plf.name})")
LOG.error(
f"error no-section provided for relocation of: {my_symbol.name} ({type(my_symbol).__name__}) ({self.plf.name} <- {other.plf.name})"
)
LOG.error(vars(relocation))
LOG.error(ext_symbol)
k = vars(ext_symbol)
@@ -317,8 +343,8 @@ class IndexedElf:
section_id = ext_symbol.section.header.sh_info
if section_id == 0:
section_id = ext_symbol.section.header.id
table.relocation(relative_offset, relocation.type, section_id, addend)
table.relocation(relative_offset, relocation.type, section_id, addend)
self.complete_relocations.add(relocation)
return True
return False
@@ -333,26 +359,30 @@ class IndexedElf:
LOG.error(f"relocation failed: {name:<14} {relocation.symbol.name}")
sys.exit(1)
def align_next(offset, alignment):
return (offset - 1 + alignment) & ~(alignment - 1)
def write_rel(path: Path,
id: int,
align: int,
bss_align: int,
bss_size: int,
name_offset: int,
name_size: int,
prolog: libelf.Symbol,
epilog: libelf.Symbol,
unresolved: libelf.Symbol,
sections: List[librel.Section],
imp_tables: List[ImpTable]):
def align_next(offset, alignment):
return (offset - 1 + alignment) & ~(alignment - 1)
def write_rel(
path: Path,
id: int,
align: int,
bss_align: int,
bss_size: int,
name_offset: int,
name_size: int,
prolog: libelf.Symbol,
epilog: libelf.Symbol,
unresolved: libelf.Symbol,
sections: List[librel.Section],
imp_tables: List[ImpTable],
):
output = librel.REL()
output.index = id
output.numSections = len(sections)
output.sectionInfoOffset = 0x4C # for version 3
output.sectionInfoOffset = 0x4C # for version 3
output.nameOffset = name_offset
output.nameSize = name_size
output.version = 3
@@ -382,7 +412,7 @@ def write_rel(path: Path,
assert output.version >= 3
with path.open('wb') as file:
with path.open("wb") as file:
librel.write_header(file, output)
sections_offset = file.tell()
@@ -396,7 +426,7 @@ def write_rel(path: Path,
if section.data:
padding = section.offset - file.tell()
if padding > 0:
file.write(b'\x00' * padding)
file.write(b"\x00" * padding)
assert section.offset == file.tell()
librel.write_section_data(file, section)
@@ -404,9 +434,9 @@ def write_rel(path: Path,
output.impOffset = file.tell()
output.impSize = len(imp_tables) * 0x8
file.write(b'\xFF' * output.impSize)
file.write(b"\xFF" * output.impSize)
output.fixSize = file.tell()
output.fixSize = file.tell()
output.relOffset = file.tell()
rel_offset = output.relOffset
for table in imp_tables:
@@ -425,7 +455,6 @@ def write_rel(path: Path,
librel.write_header(file, output)
def write_rel_from_elf(elf: IndexedElf, string_list: StringList, compress_yaz0: bool):
assert elf.id != 0
@@ -442,7 +471,7 @@ def write_rel_from_elf(elf: IndexedElf, string_list: StringList, compress_yaz0:
table.end()
# count sections
section_count = 1 # null section
section_count = 1 # null section
for elf_section in elf.plf.sections.values():
if elf_section.name == ".dead" or elf_section.name == ".rela.dead":
continue
@@ -460,7 +489,9 @@ def write_rel_from_elf(elf: IndexedElf, string_list: StringList, compress_yaz0:
if not elf_section.name in SECTION_MASK:
continue
section = librel.Section(elf_section.header.id, 0, False, False, elf_section.header.sh_size)
section = librel.Section(
elf_section.header.id, 0, False, False, elf_section.header.sh_size
)
if elf_section.header.sh_type == libelf.SHT_NOBITS:
if elf_section.header.sh_addralign >= 1:
if elf_section.header.sh_addralign >= bss_align:
@@ -474,12 +505,12 @@ def write_rel_from_elf(elf: IndexedElf, string_list: StringList, compress_yaz0:
if elf_section.header.sh_addralign >= align:
align = elf_section.header.sh_addralign
section.offset = offset
section.data = elf_section.data
if (elf_section.header.sh_flags & libelf.SHF_EXECINSTR) != 0:
section.executable_flag = True
section.executable_flag = True
offset += section.length
sections.append(section)
@@ -510,7 +541,21 @@ def write_rel_from_elf(elf: IndexedElf, string_list: StringList, compress_yaz0:
tables.append(elf.imp_tables[0])
# write the rel files
write_rel(path, elf.id, align, bss_align, bss_size, name_offset, name_size, prolog, epilog, unresolved, sections, tables)
write_rel(
path,
elf.id,
align,
bss_align,
bss_size,
name_offset,
name_size,
prolog,
epilog,
unresolved,
sections,
tables,
)
def load_elfs(str_paths):
static = None
@@ -535,7 +580,6 @@ def load_elfs(str_paths):
LOG.error(f"error: '{path}'")
LOG.error(e)
return static, plfs
tools/postprocess.py
-319
View File
@@ -1,319 +0,0 @@
#!/usr/bin/env python3
BANNER = """
# This script is the culmination of three patches supporting decompilation
# with the CodeWarrior compiler.
# - riidefi, 2020
#
# postprocess.py [args] file
#
# 1) Certain versions have a bug where the ctor alignment is ignored and set incorrectly.
# This option is enabled with -fctor-realign, and disabled by default with -fno-ctor-realign
#
# 2) Certain C++ symbols cannot be assembled normally.
# To support the buildsystem, a simple substitution system has been devised
#
# ?<ID> -> CHAR
#
# IDs (all irregular symbols in mangled names):
# 0: <
# 1: >
# 2: @
# 3: \\
# 4: ,
# 5: -
#
# This option is enabled with -fsymbol-fixup, and disabled by default with -fno-symbol-fixup
#
# 3) CodeWarrior versions below 2.3 used a different scheduler model.
# The script can currently adjust function epilogues with the old_stack option.
# -fprologue-fixup=[default=none, none, old_stack]
"""
import struct
# Substitutions
substitutions = (
('<', '_SUB_0'),
('>', '_SUB_1'),
('@', '_SUB_2'),
('\\', '_SUB_3'),
(',', '_SUB_4'),
('-', '_SUB_5')
)
def format(symbol):
for sub in substitutions:
symbol = symbol.replace(sub[0], sub[1])
return symbol
def decodeformat(symbol):
for sub in substitutions:
symbol = symbol.replace(sub[1], sub[0])
return symbol
# Stream utilities
def read_u8(f):
return struct.unpack("B", f.read(1))[0]
def read_u32(f):
return struct.unpack(">I", f.read(4))[0]
def read_u16(f):
return struct.unpack(">H", f.read(2))[0]
def write_u32(f, val):
f.write(struct.pack(">I", val))
class ToReplace:
def __init__(self, position, dest, src_size):
self.position = position # Where in file
self.dest = dest # String to patch
self.src_size = src_size # Pad rest with zeroes
# print("To replace: %s %s %s" % (self.position, self.dest, self.src_size))
def read_string(f):
tmp = ""
c = 0xff
while c != 0x00:
c = read_u8(f)
if c != 0:
tmp += chr(c)
return tmp
def ctor_realign(f, ofsSecHeader, nSecHeader, idxSegNameSeg):
patch_align_ofs = []
for i in range(nSecHeader):
f.seek(ofsSecHeader + i * 0x28)
ofsname = read_u32(f)
if not ofsname: continue
back = f.tell()
f.seek(ofsSecHeader + (idxSegNameSeg * 0x28) + 0x10)
ofsShST = read_u32(f)
f.seek(ofsShST + ofsname)
name = read_string(f)
if name == ".ctors" or name == ".dtors":
patch_align_ofs.append(ofsSecHeader + i * 0x28 + 0x20)
f.seek(back)
return patch_align_ofs
SHT_PROGBITS = 1
SHT_STRTAB = 3
def impl_postprocess_elf(f, do_ctor_realign, do_old_stack, do_symbol_fixup):
result = []
f.seek(0x20)
ofsSecHeader = read_u32(f)
f.seek(0x30)
nSecHeader = read_u16(f)
idxSegNameSeg = read_u16(f)
secF = True # First instance the section names
# Header: 0x32:
patch_align_ofs = []
if do_ctor_realign:
patch_align_ofs = ctor_realign(f, ofsSecHeader, nSecHeader, idxSegNameSeg)
for i in range(nSecHeader):
f.seek(ofsSecHeader + i * 0x28)
sh_name = read_u32(f)
sh_type = read_u32(f)
if sh_type == SHT_STRTAB and do_symbol_fixup:
if not secF:
continue
secF = False
f.seek(ofsSecHeader + i * 0x28 + 0x10)
ofs = read_u32(f)
size = read_u32(f)
f.seek(ofs)
string = ""
str_spos = ofs
for i in range(ofs, ofs+size):
c = read_u8(f)
if c == 0:
if len(string):
fixed = decodeformat(string)
if fixed != string:
result.append(ToReplace(str_spos, fixed, len(string)))
string = ""
str_spos = i+1
else:
string += chr(c)
else:
f.seek(ofsSecHeader + (idxSegNameSeg * 0x28) + 0x10)
ofsShST = read_u32(f)
f.seek(ofsShST + sh_name)
name = read_string(f)
if name == ".text" and do_old_stack:
f.seek(ofsSecHeader + i * 0x28 + 0x10)
ofs = read_u32(f)
size = read_u32(f)
# We assume
# 1) Only instructions are in the .text section
# 2) These instructions are 4-byte aligned
assert ofs != 0
assert ofs % 4 == 0
assert size % 4 == 0
f.seek(ofs)
mtlr_pos = 0
# (mtlr position, blr position)
epilogues = []
for _ in range(ofs, ofs+size, 4):
it = f.tell()
instr = read_u32(f)
# Skip padding
if instr == 0: continue
# Call analysis is not actually required
# No mtlr will exist without a blr; mtctr/bctr* is used for dynamic dispatch
# FUN_A:
# li r3, 0
# blr <---- No mtlr, move onto the next function
# FUN_B:
# ; complex function, stack manip
# mtlr r0 <---- Expect a blr
# addi r1, r1, 24
# blr <---- Confirm patch above
# mtlr alias for mtspr
if instr == 0x7C0803A6:
assert mtlr_pos == 0
mtlr_pos = it
# blr
elif instr == 0x4E800020:
if mtlr_pos:
epilogues.append((mtlr_pos, it))
mtlr_pos = 0
# Check for a lone mtlr
assert mtlr_pos == 0
# Reunify mtlr/blr instructions, shifting intermediary instructions up
for mtlr_pos, blr_pos in epilogues:
# Check if we need to do anything
if mtlr_pos + 4 == blr_pos: continue
# As the processor can only hold 6 instructions at once in the pipeline,
# it's unlikely for the mtlr be shifted up more instructions than that--usually,
# only one:
# mtlr r0
# addi r1, r1, 24
# blr
assert blr_pos - 4 > mtlr_pos
assert blr_pos - mtlr_pos <= 6 * 4
print("Patching old epilogue: %s %s" % (mtlr_pos, blr_pos))
f.seek(mtlr_pos)
mtlr = read_u32(f)
for it in range(mtlr_pos, blr_pos - 4, 4):
f.seek(it + 4)
next_instr = read_u32(f)
f.seek(it)
write_u32(f, next_instr)
f.seek(blr_pos - 4)
write_u32(f, mtlr)
return (result, patch_align_ofs)
def postprocess_elf(f, do_ctor_realign, do_old_stack, do_symbol_fixup):
patches = impl_postprocess_elf(f, do_ctor_realign, do_old_stack, do_symbol_fixup)
f.seek(0)
source_bytes = list(f.read())
for patch in patches[0]:
assert len(patch.dest) <= patch.src_size
for j in range(patch.src_size):
if j >= len(patch.dest):
c = 0
else:
c = ord(patch.dest[j])
source_bytes[patch.position + j] = c
# Patch ctor align
nP = 0
for p in patches[1]:
print("Patching ctors")
source_bytes[p + 0] = 0
source_bytes[p + 1] = 0
source_bytes[p + 2] = 0
source_bytes[p + 3] = 4
nP += 1
if nP > 1:
print("Patched ctors + dtors")
f.seek(0)
f.write(bytes(source_bytes))
def frontend(args):
inplace = ""
do_ctor_realign = False
do_old_stack = False
do_symbol_fixup = False
for arg in args:
if arg.startswith('-f'):
negated = False
if arg.startswith('-fno-'):
negated = True
arg = arg[len('-fno-'):]
else:
arg = arg[len('-f'):]
if arg == 'ctor_realign':
do_ctor_realign = not negated
elif arg == 'symbol-fixup':
do_symbol_fixup = not negated
elif arg.startswith('prologue-fixup='):
do_old_stack = arg[len('prologue-fixup='):] == 'old_stack'
else:
print("Unknown argument: %s" % arg)
elif arg.startswith('-'):
print("Unknown argument: %s. Perhaps you meant -f%s?" % (arg, arg))
else:
if inplace:
print("Cannot process %s. Only one source file may be specified." % arg)
else:
inplace = arg
if not inplace:
print("A file must be specified!")
return
try:
postprocess_elf(open(inplace, 'rb+'), do_ctor_realign, do_old_stack, do_symbol_fixup)
except FileNotFoundError:
print("Cannot open file %s" % inplace)
if __name__ == "__main__":
import sys
if len(sys.argv) < 2:
print(BANNER)
else:
frontend(sys.argv[1:])
tools/rel.py
+85 -39
View File
@@ -1,24 +1,30 @@
"""
rel.py - Tool for displaying information in .rel files
rel.py - Tool for extracting information from .rel files
"""
import click
import sys
import rich
import logging
import glob
import librel
import struct
import dataclasses
import hexdump
from libdol2asm import settings
from pathlib import Path
from collections import defaultdict
from rich.logging import RichHandler
from rich.console import Console
try:
import click
import logging
import librel
from libdol2asm import settings
from rich.logging import RichHandler
from rich.console import Console
except ImportError as e:
MISSING_PREREQUISITES = (
f"Missing prerequisite python module {e}.\n"
f"Run `python3 -m pip install --user -r tools/requirements.txt` to install prerequisites."
)
print(MISSING_PREREQUISITES, file=sys.stderr)
sys.exit(1)
VERSION = "1.0"
CONSOLE = Console()
@@ -27,26 +33,43 @@ logging.basicConfig(
level="NOTSET",
format="%(message)s",
datefmt="[%X]",
handlers=[RichHandler(console=CONSOLE, rich_tracebacks=True)]
handlers=[RichHandler(console=CONSOLE, rich_tracebacks=True)],
)
LOG = logging.getLogger("rich")
LOG.setLevel(logging.INFO)
@click.group()
@click.version_option(VERSION)
def rel():
pass
@rel.command(name="info")
@click.option('--debug/--no-debug')
@click.option('--header', '-t', 'dump_header', is_flag=True, default=False)
@click.option('--sections', '-s', 'dump_sections', is_flag=True, default=False)
@click.option('--data', '-d', 'dump_data', is_flag=True, default=False)
@click.option('--relocations', '-r', 'dump_relocation', is_flag=True, default=False)
@click.option('--imp', '-i', 'dump_imp', is_flag=True, default=False)
@click.argument("rel_path", metavar='<REL>', nargs=1, type=click.Path(exists=True,file_okay=True,dir_okay=False))
def rel_info(debug, rel_path, dump_header, dump_sections, dump_data, dump_relocation, dump_imp):
@click.option("--debug/--no-debug")
@click.option("--header", "-t", "dump_header", is_flag=True, default=False)
@click.option("--sections", "-s", "dump_sections", is_flag=True, default=False)
@click.option("--data", "-d", "dump_data", is_flag=True, default=False)
@click.option("--relocations", "-r", "dump_relocation", is_flag=True, default=False)
@click.option("--imp", "-i", "dump_imp", is_flag=True, default=False)
@click.option("--all", "-a", "dump_all", is_flag=True, default=False)
@click.argument(
"rel_path",
metavar="<REL>",
nargs=1,
type=click.Path(exists=True, file_okay=True, dir_okay=False),
)
def rel_info(
debug,
rel_path,
dump_header,
dump_sections,
dump_data,
dump_relocation,
dump_imp,
dump_all,
):
if debug:
LOG.setLevel(logging.DEBUG)
@@ -55,10 +78,20 @@ def rel_info(debug, rel_path, dump_header, dump_sections, dump_data, dump_reloca
LOG.error(f"File not found: '{path}'")
sys.exit(1)
with open(path, 'rb') as file:
with open(path, "rb") as file:
buffer = file.read()
rel = librel.read(buffer)
if dump_all:
dump_header = True
dump_sections = True
dump_data = True
dump_relocation = True
dump_imp = True
if not (dump_header or dump_sections or dump_data or dump_relocation or dump_imp):
CONSOLE.print("no dump options specified")
if dump_header:
CONSOLE.print(f"Header:")
CONSOLE.print(f"\tindex: {rel.index}")
@@ -76,32 +109,42 @@ def rel_info(debug, rel_path, dump_header, dump_sections, dump_data, dump_reloca
CONSOLE.print(f"\trel offset: 0x{rel.relOffset:04X}")
CONSOLE.print(f"\timp offset: 0x{rel.impOffset:04X}")
CONSOLE.print(f"\timp size: 0x{rel.impSize:04X}")
CONSOLE.print(f"\t_prolog: 0x{rel.prolog:04X} (section: 0x{rel.prologSection:X})")
CONSOLE.print(f"\t_epilog: 0x{rel.epilog:04X} (section: 0x{rel.epilogSection:X})")
CONSOLE.print(f"\t_unresolved: 0x{rel.unresolved:04X} (section: 0x{rel.unresolvedSection:X})")
CONSOLE.print(
f"\t_prolog: 0x{rel.prolog:04X} (section: 0x{rel.prologSection:X})"
)
CONSOLE.print(
f"\t_epilog: 0x{rel.epilog:04X} (section: 0x{rel.epilogSection:X})"
)
CONSOLE.print(
f"\t_unresolved: 0x{rel.unresolved:04X} (section: 0x{rel.unresolvedSection:X})"
)
if dump_sections:
CONSOLE.print(f"Sections:")
for i,section in enumerate(rel.sections):
CONSOLE.print(f"\t#{i:<2} offset: 0x{section.offset:08X}, length: 0x{section.length:04X}, unknown flag: {section.unknown_flag}, executable flag: {section.executable_flag}")
for i, section in enumerate(rel.sections):
CONSOLE.print(
f"\t#{i:<2} offset: 0x{section.offset:08X}, length: 0x{section.length:04X}, unknown flag: {section.unknown_flag}, executable flag: {section.executable_flag}"
)
if dump_imp:
CONSOLE.print(f"Imp Table:")
imp_buffer = rel.data[rel.impOffset:]
imp_buffer = rel.data[rel.impOffset :]
for i in range(rel.impSize // 8):
imp_offset = 0x8 * i
module_id, rel_offset = struct.unpack('>II', imp_buffer[imp_offset:][:0x8])
CONSOLE.print(f"\t#{i:<2} module: {module_id:>4}, offset: 0x{rel_offset:04X}")
module_id, rel_offset = struct.unpack(">II", imp_buffer[imp_offset:][:0x8])
CONSOLE.print(
f"\t#{i:<2} module: {module_id:>4}, offset: 0x{rel_offset:04X}"
)
if dump_relocation:
CONSOLE.print(f"Relocations:")
imp_buffer = rel.data[rel.impOffset:]
imp_buffer = rel.data[rel.impOffset :]
for i in range(rel.impSize // 8):
imp_offset = 0x8 * i
module_id, rel_offset = struct.unpack('>II', imp_buffer[imp_offset:][:0x8])
module_id, rel_offset = struct.unpack(">II", imp_buffer[imp_offset:][:0x8])
CONSOLE.print(f"\t[ module: {module_id:>4} ]")
section = None
@@ -118,13 +161,15 @@ def rel_info(debug, rel_path, dump_header, dump_sections, dump_data, dump_reloca
base += section.offset - rel.sections[1].offset
extra = f" | {base:08X} {base+relocation.offset + offset:08X}"
CONSOLE.print(f"\t#{rel_index:<3} {librel.RELOCATION_NAMES[relocation.type]:<20} {relocation.section:>4} 0x{relocation.offset+offset:08X} 0x{relocation.addend:08X}{extra}")
CONSOLE.print(
f"\t#{rel_index:<3} {librel.RELOCATION_NAMES[relocation.type]:<20} {relocation.section:>4} 0x{relocation.offset+offset:08X} 0x{relocation.addend:08X}{extra}"
)
if relocation.type == librel.R_PPC_NONE:
continue
if relocation.type == librel.R_DOLPHIN_NOP:
# NOP is used to simulate have long offset values, this is because
# the offset field is limited to 2^16-1 values. Thus, any offsets
# NOP is used to simulate long offset values, this is because
# the offset field is limited to 2^16-1 values. Thus, any offsets
# above 2^16 will be divided into a NOP + original relocation type.
offset += relocation.offset
continue
@@ -145,12 +190,13 @@ def rel_info(debug, rel_path, dump_header, dump_sections, dump_data, dump_reloca
if dump_data:
CONSOLE.print(f"Sections:")
for i,section in enumerate(rel.sections):
for i, section in enumerate(rel.sections):
if section.data:
CONSOLE.print(f"\t#{i:<2}")
hexdata = hexdump.hexdump(section.data, result='return')
hexdata = hexdump.hexdump(section.data, result="return")
CONSOLE.print(hexdata)
if __name__ == "__main__":
rel()
tools/requirements.txt
+7 -5
View File
@@ -1,8 +1,10 @@
rich
click
intervaltree
yaz0
numpy
capstone
aiofiles
GitPython
GitPython
hexdump
colorama
ansiwrap
watchdog
python-Levenshtein
cxxfilt
tools/section2cpp.py
-612
View File
@@ -1,612 +0,0 @@
#!/usr/bin/env python3
# PYTHON_ARGCOMPLETE_OK
"""
This script will extract literals and strings data
from sections located in the baserom.dol.
Useful when trying to match .rodata and .sdata2.
Usage:
./tools/section2cpp.py --section .rodata --string --object JKRSolidHeap.o
"""
import argparse
import sys
import os
import struct
from decimal import getcontext, Decimal
from pathlib import Path, PurePath, PureWindowsPath
from typing import (
Any,
Dict,
List,
Match,
NamedTuple,
NoReturn,
Optional,
Set,
Tuple,
Union,
Callable,
Pattern,
)
try:
import numpy
except:
print("error: missing numpy")
sys.exit(1)
try:
import argcomplete # type: ignore
except ModuleNotFoundError:
argcomplete = None
parser = argparse.ArgumentParser(description="Extract section data and generate C++ code.")
parser.add_argument(
"--section",
dest="section",
type=str,
metavar="SECTION",
help="SECTION to extract data from.",
required=True
)
parser.add_argument(
"--file-offset",
dest="file_offset",
type=lambda x: int(x,0),
metavar="OFFSET",
help="OFFSET in the baserom for the SECTION."
)
parser.add_argument(
"--object",
dest="object_name",
type=str,
metavar="OBJECT",
help="OBJECT filename to extract data from. (e.g. JKRSolidHeap.o)"
)
parser.add_argument(
"--map",
dest="map_path",
type=str,
metavar="MAP",
help="frameworkF.map path",
default="frameworkF.map"
)
parser.add_argument(
"--baserom",
dest="baserom",
type=str,
metavar="DOL",
help="baserom.dol path",
default="baserom.dol"
)
parser.add_argument(
"--string",
dest="as_string",
action="store_true",
help="Print arrays as strings"
)
parser.add_argument(
"--array",
dest="as_array",
action="store_true",
help="Print everything as u8 arrays"
)
parser.add_argument(
"--shift-jis",
dest="shift_jis",
action="store_true",
help="Convert shift-jis to utf-8"
)
#
#
#
def _itersplit(l, splitters):
current = []
for item in l:
if item in splitters:
yield current
current = []
else:
current.append(item)
yield current
def magicsplit(l, *splitters):
return [subl for subl in _itersplit(l, splitters) ]
def str_encoding(data):
if data[-1] != 0:
return None
try:
data.decode("utf-8")
return "utf-8"
except:
pass
try:
data.decode("shift_jisx0213")
return "shift-jis"
except:
pass
return None
def encoding_char_list(encoding, data):
if args.shift_jis and encoding == "shift-jis":
try:
return escape(data.decode("shift_jisx0213"))
except:
pass
return [ str(bytes([x]))[2:-1].replace("\"", "\\\"") for x in data ]
def raw_string(data):
return "".join(data)
def raw_array(data):
return ",".join([hex(x) for x in list(data)])
def escape_char(v):
if v == "\n":
return "\\n"
elif v == "\t":
return "\\t"
elif v == "\v":
return "\\v"
elif v == "\b":
return "\\b"
elif v == "\r":
return "\\r"
elif v == "\f":
return "\\f"
elif v == "\a":
return "\\a"
elif v == "\\":
return "\\\\"
elif v == "\"":
return "\\\""
elif ord(v) < 32 and ord(v) > 127:
return "\\x" + hex(v)[2:].upper().rjust(2, '0')
else:
return v
def escape(v):
return "".join([ escape_char(x) for x in list(v) ])
def bytes2float32(data):
if len(data) < 4:
return None
result = numpy.frombuffer(data[0:4][::-1], dtype='float32')
if result:
return result[0]
else:
return None
def bytes2float64(data):
if len(data) < 8:
return None
result = numpy.frombuffer(data[0:8][::-1], dtype='float64')
if result:
return result[0]
else:
return None
def is_nice_float32(f):
try:
if int(f*1000) == f*1000:
return True
if int(f*100) == f*100:
return True
if int(f*10) == f*10:
return True
if int(f) == f:
return True
except:
return False
return False
def is_nice_float64(f):
try:
if int(f*1000) == f*1000:
return True
if int(f*100) == f*100:
return True
if int(f*10) == f*10:
return True
if int(f) == f:
return True
except:
return False
return False
float32_exact: Dict[numpy.float32, Tuple[int,int]] = {}
float64_exact: Dict[numpy.float64, Tuple[int,int]] = {}
getcontext().prec = 64
for i in range(1,32):
for j in range(1,32):
if i%j == 0:
continue
d = Decimal(i)/Decimal(j)
f = numpy.float32(d)
if str(f) != str(d):
if not f in float32_exact:
float32_exact[f] = (i,j)
for i in range(1,32):
for j in range(1,32):
if i%j == 0:
continue
d = Decimal(i)/Decimal(j)
f = numpy.float64(d)
if str(f) != str(d):
if not f in float64_exact:
float64_exact[f] = (i,j)
class Symbol:
def __init__(self, name, addr, size):
self.name = name
self.addr = addr
self.size = size
self.padding = 0
def __str__(self):
return " %s %s %s+%s %s" % (self.name.ljust(40, ' '), hex(self.addr), hex(self.addr + self.size), hex(self.padding), hex(self.size))
class ObjectFile:
def __init__(self, path):
self.path = path
self.symbols = []
self.start = 0
self.end = 0
self.mk = False
def addSymbol(self, name, str_addr, str_size):
addr = int(str_addr, base=16)
size = int(str_size, base=16)
symbol = Symbol(name, addr, size)
if not self.symbols:
self.start = symbol.addr
else:
last_symbol = self.symbols[-1]
last_addr = last_symbol.addr + last_symbol.size
if last_addr != addr:
last_symbol.padding += addr - last_addr
self.symbols += [ symbol ]
def setEnd(self, end):
self.end = end
last_symbol = self.symbols[-1]
last_symbol.padding = self.end - (last_symbol.addr + last_symbol.size)
def find_symbols():
file = map_path.open('r')
lines = file.readlines()
in_section = False
last_obj = None
for line in lines:
data = [ x.strip() for x in line.strip().split(" ") ]
data = [ x for x in data if len(x) > 0 ]
if len(data) == 3:
in_section = False
if data[0] == section:
in_section = True
continue
if not in_section:
continue
if len(data) < 6 or len(data) > 7:
continue
# get object filename
obj = data[5]
if len(data) > 6:
obj = data[6]
# remove path from object filename
obj = obj.split("\\")[-1]
if last_obj != obj:
assert obj not in object_map
object_map[obj] = ObjectFile(obj)
last_obj = obj
# add symbol
size = data[1]
addr = data[2]
name = data[4]
object_map[obj].addSymbol(name, addr, size)
keys = list(object_map.keys())
for i,_ in enumerate(keys[:-1]):
obj = object_map[keys[i]]
next_obj = object_map[keys[i + 1]]
obj.setEnd(next_obj.start)
# total size of rodata must be aligned to 0x20
obj = object_map[keys[-1]]
last_symbol = obj.symbols[-1]
last_addr = last_symbol.addr + last_symbol.size
last_symbol.padding = ((last_addr + 31) & ~31) - last_addr
file.close()
def chunks(lst, n):
for i in range(0, len(lst), n):
yield lst[i:i + n]
def data_as_string(data):
return ", ".join([ "0x" + hex(x)[2:].rjust(2, '0') for x in data ])
class Literal:
def __init__(self, name, type, value, comment=None):
self.name = name
self.type = type
self.value = value
self.comment = comment
def format(self):
return str(self.value)
def lines(self):
line = "static const %s %s = %s;" % (self.type, self.name, self.format())
if self.comment:
line = line.ljust(90, ' ') + " // " + self.comment
return [ line ]
def __str__(self):
return "\n".join(self.lines())
class Label(Literal):
def __init__(self, name):
super().__init__(name, "", None, None)
def lines(self):
return [ "", "", "// " + self.name ]
class Float32Literal(Literal):
def __init__(self, name, value, comment=None):
super().__init__(name, "float", value, comment)
def format(self):
return "%sf" % self.value
class Float64Literal(Literal):
def __init__(self, name, value, comment=None):
super().__init__(name, "double", value, comment)
class FractionFloat32Literal(Literal):
def __init__(self, name, value, comment=None):
super().__init__(name, "float", value, comment)
def format(self):
return "%i.0f / %i.0f" % self.value
class FractionFloat64Literal(Literal):
def __init__(self, name, value, comment=None):
super().__init__(name, "double", value, comment)
def format(self):
return "%i.0 / %i.0" % self.value
class U32Literal(Literal):
def __init__(self, name, value, comment=None):
super().__init__(name, "u32", value, comment)
class S32Literal(Literal):
def __init__(self, name, value, comment=None):
super().__init__(name, "s32", value, comment)
class S64Literal(Literal):
def __init__(self, name, value, comment=None):
super().__init__(name, "s64", value, comment)
class U64Literal(Literal):
def __init__(self, name, value, comment=None):
super().__init__(name, "u64", value, comment)
class ArrayLiteral(Literal):
def __init__(self, name, value, comment=None):
super().__init__(name, "u8", value, comment)
def lines(self):
one_line = "static const %s %s[%i] = { %s };" % (self.type, self.name, len(self.value), data_as_string(self.value))
lines = []
if len(one_line) < 90:
lines += [ one_line ]
else:
lines += [ "static const %s %s[%i] = {" % (self.type, self.name, len(self.value)) ]
data_chunks = chunks(list(self.value), 16)
for chunk in data_chunks:
lines += [ " " + data_as_string(chunk) ]
lines += [ "};" ]
if lines and self.comment:
lines[0] = lines[0].ljust(90, ' ') + " // " + self.comment
return lines
class StringLiteral(Literal):
def __init__(self, name, encoding, value, comment=None):
assert value[-1] == 0
super().__init__(name, "char", value[:-1], comment)
self.encoding = encoding
def lines(self):
char_list = encoding_char_list(self.encoding, self.value)
one_line = "static const %s %s = \"%s\";" % (self.type, self.name, raw_string(char_list))
lines = []
if len(one_line) < 90:
lines += [ one_line ]
else:
lines += [ "static const %s %s = " % (self.type, self.name) ]
data_chunks = chunks(char_list, 16)
for chunk in data_chunks:
lines += [ " \"%s\"" % raw_string(chunk) ]
lines[-1] += ";"
if lines and self.comment:
lines[0] = lines[0].ljust(90, ' ') + " // " + self.comment
return lines
def output_cpp():
object_names = []
if object_name:
if not object_name in object_map:
print("error: %s object file not found!" % object_name)
sys.exit(1)
object_names += [ object_name ]
else:
object_names = [*object_map.keys()]
br = baserom.open("rb")
br.seek(0, os.SEEK_END)
br_size = br.tell()
br.seek(0, os.SEEK_SET)
literals = []
for obj_name in object_names:
literals += [ Label(obj_name) ]
obj = object_map[obj_name]
for symbol in obj.symbols:
label = "lbl_%s" % (hex(symbol.addr).upper()[2:])
symbol_file_offset = symbol.addr - file_offset
symbol_file_size = symbol.size + symbol.padding
if symbol_file_offset + symbol_file_size > br_size:
print("error: reading outside baserom file. (%i, %i)" % (symbol_file_offset + symbol_file_size, br_size))
br.seek(symbol_file_offset, os.SEEK_SET)
data = br.read(symbol.size)
padding = br.read(symbol.padding)
if args.as_string:
offset = 0
str_segments = [ x for x in magicsplit(data, 0) ]
for segment in str_segments[:-1]:
str_data = bytes(segment + [0])
encoding = str_encoding(str_data)
str_label = "lbl_%s" % (hex(symbol.addr + offset).upper()[2:])
if encoding == "shift-jis":
literals += [ StringLiteral(str_label, "shift-jis", str_data, "TODO: shift-jis strings in Metrowerks") ]
elif encoding == "utf-8":
literals += [ StringLiteral(str_label, "utf-8", str_data) ]
else:
literals += [ ArrayLiteral(str_label, str_data, "undecodable string") ]
offset += len(str_data)
if padding:
padding_label = "lbl_%s" % (hex(symbol.addr + symbol.size).upper()[2:])
literals += [ StringLiteral(padding_label, None, padding, "padding") ]
padding = None
elif args.as_array:
literals += [ ArrayLiteral(label, data) ]
else:
lit = None
if len(data) == 4:
u32_data = struct.unpack('>I', data)[0]
s32_data = struct.unpack('>i', data)[0]
float_data = bytes2float32(data)
if s32_data == 0 or (s32_data >= -4096 and s32_data <= 4096):
lit = S32Literal(label, s32_data)
elif u32_data == 0 or (u32_data < 4096):
lit = U32Literal(label, u32_data)
elif float_data in float32_exact:
lit = FractionFloat32Literal(label, float32_exact[float_data], "%sf %s" % (float_data, hex(u32_data)))
elif is_nice_float32(float_data):
lit = Float32Literal(label, float_data, hex(u32_data))
elif len(data) == 8:
u64_data = struct.unpack('>Q', data)[0]
s64_data = struct.unpack('>q', data)[0]
double_data = bytes2float64(data)
if u64_data == 0x4330000000000000:
lit = Float64Literal(label, double_data, "%s | u32 to float (compiler-generated)" % hex(u64_data))
elif u64_data == 0x4330000080000000:
lit = Float64Literal(label, double_data, "%s | s32 to float (compiler-generated)" % hex(u64_data))
elif s64_data == 0 or (s64_data >= -4096 and s64_data <= 4096):
lit = S64Literal(label, s64_data)
elif u64_data == 0 or (u64_data < 4096):
lit = U64Literal(label, u64_data)
elif double_data in float64_exact:
lit = FractionFloat64Literal(label, float64_exact[double_data], "%s %s" % (double_data, hex(u64_data)))
elif is_nice_float64(double_data):
lit = Float64Literal(label, double_data, hex(u64_data))
if not lit:
lit = ArrayLiteral(label, data)
literals += [ lit ]
if padding:
padding_label = "lbl_%s" % (hex(symbol.addr + symbol.size).upper()[2:])
literals += [ ArrayLiteral(padding_label, padding, "padding") ]
for lit in literals:
print(lit)
br.close()
#
#
#
try:
args = parser.parse_args()
except:
parser.print_help()
sys.exit(0)
section = args.section
object_name = args.object_name
file_offset: Optional[int] = args.file_offset
baserom = Path(args.baserom)
map_path = Path(args.map_path)
file_offsets = {
".rodata": 0x80003000,
".sdata": 0x800802A0,
".sdata2": 0x800811A0,
}
if not file_offset:
if not section in file_offsets:
print("error: missing --file-offset")
sys.exit(1)
else:
file_offset = file_offsets[section]
if not baserom.exists():
print("error: baserom '%s' not found!" % args.baserom)
sys.exit(1)
if not map_path.exists():
print("error: frameworkF.map '%s' not found!" % args.map_path)
sys.exit(1)
object_map: Dict[str,ObjectFile] = {}
find_symbols()
output_cpp()
tools/splitter/asm_parser.py
-159
View File
@@ -1,159 +0,0 @@
from dataclasses import dataclass
from parsy import string, regex, seq, generate, line_info
from typing import Optional, List, Union, Protocol
class Emittable(Protocol):
def emit(self) -> str:
...
@dataclass
class BlockComment:
text: str
def emit(self) -> str:
return f'/*{self.text}*/'
@dataclass
class TrailingComment:
text: str
def emit(self) -> str:
return f'# {self.text}'
@dataclass
class Include:
file: str
def emit(self) -> str:
return f'.include "{self.file}"'
@dataclass
class Section:
name: str
flags: Optional[str]
def emit(self) -> str:
directive = f'.section .{self.name}'
if self.flags is not None:
directive += f', "{self.flags}"'
return directive
@dataclass
class Global:
symbol: str
def emit(self) -> str:
return f'.global {self.symbol}'
@dataclass
class Label:
symbol: str
def emit(self) -> str:
return f'{self.symbol}:'
@dataclass
class Instruction:
opcode: str
operands: List[str]
def emit(self) -> str:
instr = self.opcode
if len(self.operands) > 0:
instr += ' ' + ', '.join(self.operands)
return instr
@dataclass
class Line:
index: int
content: List[
Union[
BlockComment, TrailingComment, Instruction, Global, Section, Include, Label
]
]
body: Optional[Union[Global, Section, Include, Label, Instruction]]
def emit(self) -> str:
return ' '.join([x.emit() for x in self.content])
space = regex(r'[ \t]+')
line_ending = regex('(\n)|(\r\n)').desc('newline')
pad = regex(r'[ \t]*')
block_comment = (
string('/*') >> regex(r'[\w\s]*').map(BlockComment) << string('*/')
).desc('block comment')
trailing_comment = (
string('#') >> pad >> regex(r'[^\n\r]*').map(TrailingComment)
).desc('trailing comment')
symbolname = regex(r'[a-zA-Z._$][a-zA-Z0-9._$?]*')
label = (symbolname.map(Label) << string(':')).desc('label')
delimited_string = (string('"') >> regex(r'[^"]*') << string('"')).desc(
'double-quote delimited string'
)
directive_include = string('include') >> space >> delimited_string.map(Include)
directive_section = seq(
name=string('section')
>> space
>> string('.')
>> regex(r'[a-z]+'),
flags=(pad >> string(',') >> space >> delimited_string).optional(),
).combine_dict(Section)
directive_global = string('global') >> space >> symbolname.map(Global)
directive = (
string('.')
>> (
directive_include
| directive_section
| directive_global
| string('text').result(Section('text', flags=None))
| string('data').result(Section('data', flags=None))
)
).desc('directive')
opcode = regex(r'[a-z_0-9]+\.?').concat().desc('opcode')
operand = regex(r'[^,#\s]+')
operands = operand.sep_by(string(',') << pad)
@generate
def instruction():
op = yield opcode
sp = yield space.optional()
if sp:
oprs = yield operands
else:
oprs = []
return Instruction(op, oprs)
@generate
def line():
line, _ = yield line_info
content = yield (pad >> block_comment << pad).many()
body = yield (directive | label | instruction).optional() << pad
if body:
content.append(body)
content += yield (pad >> block_comment).many()
trailing = yield (pad >> trailing_comment).optional()
if trailing:
content.append(trailing)
return Line(line, content, body)
asm = line.sep_by(line_ending)
tools/splitter/demangle.py
-311
View File
@@ -1,311 +0,0 @@
from typing import List, Optional
from pathlib import Path
from dataclasses import dataclass, field
import re
operator_func_re = re.compile(r'^__([a-z]+)')
types = {
'i': 'int',
'l': 'long',
's': 'short',
'c': 'char',
'f': 'f32',
'd': 'f64',
'v': 'void',
'x': 'long long',
'b': 'bool',
'e': 'varargs...',
}
short_type_names = {
'char': '8',
'short': '16',
'long' : '32',
'long long': '64',
}
# {'defctor', 'ops',}
special_funcs = {
'eq': 'operator==',
'as': 'operator=',
'ne': 'operator!=',
'dv': 'operator/',
'pl': 'operator+',
'mi': 'operator-',
'ml': 'operator*',
'adv': 'operator/=',
'apl': 'operator+=',
'ami': 'operator-=',
'amu': 'operator*=',
'lt': 'operator<',
'gt': 'operator>',
'cl': 'operator()',
'dla': 'operator delete[]',
'nwa': 'operator new[]',
'dl': 'operator delete',
'nw': 'operator new',
}
@dataclass
class Param:
name: str = ''
pointer_lvl: int = 0
is_const: bool = False
is_ref: bool = False
is_unsigned: bool = False
is_signed: bool = False
def to_str(self) -> str:
ret = ''
if self.is_const:
ret += 'const '
if self.name in short_type_names:
ret += 'u' if self.is_unsigned else 's'
ret += short_type_names[self.name]
else:
if self.is_unsigned:
ret += 'unsigned '
ret += self.name
for _ in range(self.pointer_lvl):
ret += '*'
if self.is_ref:
ret += '&'
return ret
@dataclass
class FuncParam:
ret_type: Optional[str] = None
params: List[str] = field(default_factory=list)
def to_str(self) -> str:
ret = ''
if self.ret_type is None:
ret += 'void'
else:
ret += self.ret_type
ret += ' (*)('
ret += ', '.join(self.params)
ret += ')'
return ret
class ParseError(Exception):
...
class ParseCtx:
def __init__(self, mangled: str):
self.mangled = mangled
self.index = 0
self.demangled = []
self.cur_type = None
self.class_name = None
self.is_const = False
self.func_name = None
def demangle(self):
# this split is still not accurate, but good enough for most cases
last_f = self.mangled.rfind('F')
if last_f == -1:
return
split_pos = self.mangled.rfind('__', 0, last_f)
if split_pos == -1 or split_pos == 0:
return
self.func_name = self.mangled[:split_pos]
self.mangled = self.mangled[split_pos+2:]
if self.func_name.startswith('__'):
match = operator_func_re.match(self.func_name)
if match:
special_func_name = match.group(1)
if special_func_name in special_funcs:
self.func_name = special_funcs[special_func_name]
else:
if special_func_name == 'ct':
self.func_name = '.ctor'
elif special_func_name == 'dt':
self.func_name = '.dtor'
self.demangle_first_class()
while self.index < len(self.mangled):
self.demangled.append(self.demangle_next_type())
if self.func_name == '.ctor':
self.func_name = self.class_name
if self.func_name == '.dtor':
self.func_name = '~' + self.class_name
def demangle_first_class(self):
if self.peek_next_char().isdecimal():
self.class_name = self.demangle_class()
if self.peek_next_char() == 'C':
self.is_const = True
self.index += 1
assert self.consume_next_char() == 'F', 'next char should be F!'
elif self.peek_next_char() == 'Q':
self.index += 1
self.class_name = self.demangle_qualified_name()
if self.peek_next_char() == 'C':
self.is_const = True
self.index += 1
assert self.consume_next_char() == 'F', 'next char should be F!'
else:
assert self.consume_next_char() == 'F', 'next char should be F!'
def demangle_next_type(self) -> str:
cur_type = Param()
while True:
cur_char = self.peek_next_char()
if cur_char.isdecimal():
class_name = self.demangle_class()
cur_type.name = class_name
return cur_type.to_str()
elif cur_char in types:
type_name = self.demangle_prim_type()
cur_type.name = type_name
return cur_type.to_str()
elif cur_char == 'U':
cur_type.is_unsigned = True
self.index += 1
elif cur_char == 'S':
cur_type.is_signed = True
self.index += 1
elif cur_char == 'C':
cur_type.is_const = True
self.index += 1
elif cur_char == 'P':
cur_type.pointer_lvl += 1
self.index += 1
elif cur_char == 'R':
cur_type.is_ref = True
self.index += 1
elif cur_char == 'F':
self.index += 1
func = self.demangle_function()
return func.to_str()
elif cur_char == 'Q':
self.index += 1
return self.demangle_qualified_name()
elif cur_char == 'A':
if cur_type.pointer_lvl < 1 and not cur_type.is_ref:
raise ParseError("pointer level for array is wrong!")
# decrease pointer level by one, cause one is already handled in the array demangle
if not cur_type.is_ref:
cur_type.pointer_lvl -= 1
cur_type.name = self.demangle_array()
return cur_type.to_str()
else:
raise ParseError(f'unexpected character {cur_char}')
def demangle_array(self) -> str:
sizes = []
while self.peek_next_char() == 'A':
self.index += 1
sizes.append(self.read_next_int())
if self.consume_next_char() != '_':
raise ParseError("Need to have '_' after Array size!")
array_type = self.demangle_next_type()
return f'{array_type} []' + ''.join(f'[{i}]' for i in sizes)
def demangle_function(self) -> FuncParam:
func_param = FuncParam()
while True:
cur_char = self.peek_next_char()
if cur_char == '_':
self.index += 1
func_param.ret_type = self.demangle_next_type()
return func_param
func_param.params.append(self.demangle_next_type())
def demangle_qualified_name(self) -> str:
part_count = int(self.consume_next_char())
parts = []
for _ in range(part_count):
parts.append(self.demangle_class())
return '::'.join(parts)
def read_next_int(self) -> int:
class_len_str = ''
cur_char = self.peek_next_char()
while cur_char.isdecimal():
class_len_str += cur_char
self.index += 1
cur_char = self.peek_next_char()
return int(class_len_str)
def demangle_class(self) -> str:
if not self.peek_next_char().isdecimal():
raise ParseError(f'class mangling must start with number')
class_len = self.read_next_int()
class_name = self.mangled[self.index : self.index + class_len]
self.index += class_len
if self.peek_next_char() == 'M':
self.index += 1
class_name += '::' + self.demangle_class()
return class_name
def demangle_prim_type(self) -> str:
ret = types[self.consume_next_char()]
return ret
def consume_next_char(self) -> str:
next_char = self.mangled[self.index]
self.index += 1
return next_char
def peek_next_char(self) -> str:
if self.index >= len(self.mangled):
return None
return self.mangled[self.index]
def to_str(self) -> str:
if self.func_name is None:
return ''
elif self.class_name is None:
return self.func_name + '(' + ', '.join(self.demangled) + ')'
else:
return self.class_name + '::' + self.func_name + '(' + ', '.join(self.demangled) + ')' + (' const' if self.is_const else '')
def demangle(s):
p = ParseCtx(s)
p.demangle()
return p.to_str()
def parse_framework_map(path: Path):
address_funcname = {}
with path.open() as f:
for line in f.readlines():
if line.startswith('.ctors'):
return address_funcname
if not line.startswith(' '):
continue
funcname = line[30:].split(' ', 1)[0]
address = line[18:26]
address_funcname[address] = funcname
return address_funcname
# def try_demangle_all():
# with open('frameworkF.map') as f:
# for line in f.readlines():
# if line.startswith('.ctors'):
# return
# if not line.startswith(' '):
# continue
# line = line[30:]
# line_spl = line.split(' ',1)[0]
# try:
# d = demangle(line_spl)
# if d:
# print(d)
# # except NotImplementedError:
# # pass
# except Exception as e:
# # print(f'could not demangle {line_spl}: {repr(e)}')
# # raise e
# pass
# try_demangle_all()
tools/splitter/split.py
-338
View File
@@ -1,338 +0,0 @@
"""
split.py - 202x erin moon for zeldaret
"""
from typing import Iterable, List
from dataclasses import dataclass
from pathlib import Path, PosixPath
from textwrap import dedent
from loguru import logger
from datetime import datetime
import re
import click
from asm_parser import asm, Emittable, Global, Label, Line, BlockComment, Instruction
from demangle import parse_framework_map, demangle
from util import PathPath, pairwise
from pprint import pprint
import pickle
import IPython
SDA_BASE = 0x80458580
SDA2_BASE = 0x80459A00
__version__ = 'v0.4'
def function_global_search(lines: List[Line]) -> Iterable[Line]:
i = 0
while i < len(lines):
if isinstance(lines[i].body, Global):
sym = lines[i].body.symbol
if isinstance(lines[i + 1].body, Label) and lines[i + 1].body.symbol == sym:
yield lines[i]
i += 2
else:
i += 1
def emit_lines(lines: List[Line]) -> str:
return '\n'.join([line.emit() for line in lines])
def comment_out(line: Line) -> Line:
return Line(line.index, [BlockComment(line.emit())], None)
def fix_sda_base_add(line: Line) -> Line:
if 'SDA' in line.body.operands[2]:
ops = line.body.operands[2].split('-')
lbl_name = ops[0]
if ops[1] == '_SDA_BASE_':
sda_reg = 'r13'
elif ops[1] == '_SDA2_BASE_':
sda_reg = 'r2'
else:
logger.error('Unknown SDABASE!')
return line
line.body.opcode = 'la'
line.body.operands = [line.body.operands[0], f'{lbl_name}({sda_reg})']
return line
QUANT_REG_RE = re.compile(r'qr(\d+)')
def patch_gqrs(line: Line) -> Line:
line.body.operands = [QUANT_REG_RE.sub(r'\1', o) for o in line.body.operands]
return line
@dataclass
class Function:
name: str
addr: int
lines: List[Line]
@property
def line_count(self):
return len(self.lines)
@property
def filename(self) -> str:
return f'func_{self.addr:X}.s'
def include_path(self, base: Path) -> str:
return str(PosixPath(base) / PosixPath(self.filename))
def find_functions(lines: List[Line], framework_map) -> Iterable[Function]:
for func_global_line, next_func_global_line in pairwise(
function_global_search(lines)
):
# some blocks weren't properly split, use the map to find missing functions
fr = func_global_line.index + 2
# if no next global, to = None => end of file
to = next_func_global_line and next_func_global_line.index
func_lines = lines[fr:to]
func_idx = []
for idx, line in enumerate(func_lines):
if isinstance(line.body, Instruction):
addr = int(line.content[0].text.strip().split()[0], 16)
if f'{addr:x}' in framework_map:
func_idx.append(idx)
for start_idx, end_idx in pairwise(func_idx):
sub_func_lines = func_lines[
start_idx : (len(func_lines) if end_idx == None else end_idx)
]
addr = int(sub_func_lines[0].content[0].text.strip().split()[0], 16)
yield Function(
name=func_global_line.body.symbol
if start_idx == 0
else f'func_{addr:X}',
addr=addr,
lines=sub_func_lines,
)
def emit_cxx_asmfn(inc_base: Path, func: Function) -> str:
return dedent(
'''\
asm void {name}(void) {{
nofralloc
#include "{inc}"
}}'''.format(
name=func.name, inc=func.include_path(inc_base)
)
)
def emit_cxx_extern_fns(tu_file: str, labels: Iterable[str]) -> str:
def decl(label):
return f'void {label}(void);'
defs = '\n '.join(decl(label) for label in sorted(labels))
return (
f'// additional symbols needed for {tu_file}\n'
f'// autogenerated by split.py {__version__} at {datetime.utcnow()}\n'
'extern "C" {\n'
' ' + defs + '\n}'
)
def emit_cxx_extern_vars(tu_file: str, labels: Iterable[str]) -> str:
def decl(label):
return f'extern u8 {label};'
return (
f'// additional symbols needed for {tu_file}\n'
f'// autogenerated by split.py {__version__} at {datetime.utcnow()}\n'
+ '\n'.join(decl(label) for label in sorted(labels))
+ '\n'
)
@click.command()
@click.argument('src', type=PathPath(file_okay=True, dir_okay=False, exists=True))
@click.argument('cxx_out', type=PathPath(file_okay=True, dir_okay=False))
@click.option(
'--funcs-out',
type=PathPath(file_okay=False, dir_okay=True),
default='include/funcs',
)
@click.option('--s-include-base', type=str, default='funcs')
@click.option(
'--framework-map-file',
type=PathPath(file_okay=True, dir_okay=False),
default='frameworkF.map',
)
@click.option(
'--ldscript-file',
type=PathPath(file_okay=True, dir_okay=False),
default='ldscript.lcf',
)
@click.option('--from-line', type=int)
@click.option('--to-line', type=int)
@click.option('--preparsed', is_flag=True)
@click.option('--forceactive',
type=click.Choice(['all', 'none', 'missingfunc']), default='missingfunc')
def split(
src,
cxx_out,
funcs_out,
s_include_base,
framework_map_file,
ldscript_file,
from_line,
to_line,
preparsed,
forceactive
):
funcs_out.mkdir(exist_ok=True, parents=True)
if preparsed:
logger.info('loading preparsed assembly')
with src.open('rb') as f:
lines = pickle.load(f)
else:
logger.info('parsing assembly')
lines = asm.parse(src.read_text())
lines = lines[
(from_line - 1 if from_line else 0) : (to_line - 1 if to_line else -1)
]
logger.info('parsing map file')
framework_map = parse_framework_map(framework_map_file)
logger.debug(f'loaded {len(framework_map)} symbols from map')
logger.info('reading ldscript')
ldscript_file_content = ldscript_file.read_text()
new_ldfuncs = []
# -- get all defined labels and jump targets
jumped_labels = set()
defined_labels = set()
logger.info('scanning for branch targets')
for line in lines:
if isinstance(line.body, Label):
defined_labels.add(line.body.symbol)
if isinstance(line.body, Instruction):
if line.body.opcode[0] == 'b' and line.body.operands != []: # branch
jumped_labels.add(line.body.operands[0]) # jump target
# -- find everything of the form lbl_[hex] that's in an operand on the RHS of a l* instruction
# this is a relatively okay assumption given that any var that's *not* of the form lbl_ has
# probably already been renamed and thus is exported in variables.h
LBL_RE = re.compile(r'lbl_[0-9A-F]+')
def find_labels_in_operands(operands):
for operand in operands:
if match := LBL_RE.search(operand):
yield match.group()
logger.info('scanning for load/store target labels')
loaded_labels = set()
for line in lines:
if isinstance(line.body, Instruction):
if line.body.opcode[0] in {'l', 's'} or line.body.opcode == 'addi': # load and store instructions, ish. Also addi for loading SDA addresses
loaded_labels |= set(find_labels_in_operands(line.body.operands))
# -- find all defined functions and split them
functions = list(find_functions(lines, framework_map))
logger.info('splitting functions')
for func in functions:
logger.debug(
f'working on function {func.name} @ {func.addr:X} with {func.line_count} lines'
)
# comment out .globals
func.lines = [
comment_out(line) if isinstance(line.body, Global) else line
for line in func.lines
]
# fix SDA_BASE addi
func.lines = [
fix_sda_base_add(line)
if isinstance(line.body, Instruction)
and line.body.opcode == 'addi'
else line
for line in func.lines
]
# remove GQR mnemonics
func.lines = [
patch_gqrs(line)
if isinstance(line.body, Instruction)
and line.body.opcode.startswith('psq_')
else line
for line in func.lines
]
# check if needs to be defined in ldscript
if forceactive != 'none':
if not func.name in ldscript_file_content and (forceactive=='all' or func.name.startswith('func_')):
new_ldfuncs.append(func.name)
with open(out_path := funcs_out / func.filename, 'w') as f:
logger.debug(f'emitting {out_path}')
f.write(emit_lines(func.lines))
# -- dump new labels to functions.h
logger.info('dumping labels to extern functions header')
func_labels = jumped_labels - defined_labels
# add in everything we def to make sure asm can get backrefs
for func in functions:
func_labels.add(func.name)
# -- write asm stubs to cxx_out (could've done this as part of previous loop but imo this is cleaner)
logger.info(f'emitting c++ asm stubs to {cxx_out}')
with open(cxx_out, 'w') as f:
f.write(
f'/* {cxx_out.name} autogenerated by split.py {__version__} at {datetime.utcnow()} */\n\n'
)
f.write('#include "global.h"\n\n')
# extern functions
f.write(emit_cxx_extern_fns(cxx_out.name, func_labels))
f.write('\n\n')
# extern variables
f.write(emit_cxx_extern_vars(cxx_out.name, loaded_labels))
f.write('\n\n')
f.write('extern "C" {\n')
for func in functions:
logger.debug(f'emitting asm stub for {func.name}')
mangled_func_name = framework_map[f'{func.addr:x}']
f.write(f'// {mangled_func_name}\n')
try:
demangled_func_name = demangle(mangled_func_name)
f.write(f'// {demangled_func_name}\n')
except Exception as e:
logger.warning(f"could not demangle symbol '{mangled_func_name}': {e}")
f.write(emit_cxx_asmfn(s_include_base, func))
f.write('\n\n')
f.write('};\n') # extern C end
# -- make defined functions FORCEACTIVE in ldscript.lcf
logger.info(f'writing to FORCEACTIVE in linker script')
forceactive_start = ldscript_file_content.find('FORCEACTIVE')
forceactive_end = ldscript_file_content.find('}', forceactive_start)
for func in new_ldfuncs:
ldscript_file_content = (
ldscript_file_content[:forceactive_end]
+ func
+ '\n'
+ ldscript_file_content[forceactive_end:]
)
ldscript_file.write_text(ldscript_file_content)
if __name__ == '__main__':
split()
tools/splitter/util.py
-17
View File
@@ -1,17 +0,0 @@
import itertools
import click
from pathlib import Path
def pairwise(iterable):
"s -> (s0,s1), (s1,s2), (s2, s3), ..."
a, b = itertools.tee(iterable)
next(b, None)
return itertools.zip_longest(a, b)
class PathPath(click.Path):
"""A Click path argument that returns a pathlib Path, not a string"""
def convert(self, value, param, ctx):
return Path(super().convert(value, param, ctx))
tools/tp.py
+753 -251
View File
File diff suppressed because it is too large Load Diff
tools/vtables.py
-33
View File
@@ -1,33 +0,0 @@
#!/usr/bin/env python3
BANNER = """
# This script is will go through the frameworkF.map and extract
# all vtables and their location. This information can be used to
# place the compiler generated vtables in their correct place.
# - Julgodis, 2020
"""
def extract():
file = open('frameworkF.map', 'r')
lines = file.readlines()
output = open('vtable.lcf', 'w')
for line in lines:
data = [ x.strip() for x in line.strip().split(" ") ]
data = [ x for x in data if len(x) > 0 ]
if len(data) < 6 or len(data) > 7:
continue
if not data[4].startswith("__vt"):
continue
output.write("\"%s\" = %s;\n" % (data[4], "0x" + data[2]))
output.close()
if __name__ == "__main__":
import sys
print(BANNER)
print("...")
extract()
print("COMPLETE vtable.lcf")