From b7a609b9e1a214f5f5fb68ba6840a61a40113734 Mon Sep 17 00:00:00 2001 From: Prakxo Date: Thu, 16 Mar 2023 21:38:21 +0100 Subject: [PATCH] fix linux builds --- README.MD | 58 +- common.py | 825 +++++++++++------------ configure.py | 1771 +++++++++++++++++++++++++------------------------- tools/ppcdis | 2 +- 4 files changed, 1318 insertions(+), 1338 deletions(-) diff --git a/README.MD b/README.MD index 2186aa74..106e30ef 100644 --- a/README.MD +++ b/README.MD @@ -1,29 +1,29 @@ -# Animal Crossing Decompilation - -Decompilation in progress of Animal Crossing (GAFE01) - -## Cloning - -Use `--recursive` when cloning to have ppcdis in the repository. - -## Build - -**Currently, it has only been tested with [wibo](https://github.com/decompals/wibo) under WSL2, but should build under other platforms and with Wine.** - -### Building - -- Dump a copy of the game and extract **main.dol** and **foresta.rel.szs**. -- Decompress **foresta.rel.szs** with yaz0 found in *tools/*. -- Place **main.dol** and **foresta.rel** in *dump/*. -- Place CodeWarrior 1.3.2 in *tools/1.3.2/* and 1.2.5 in *tools/1.2.5/*. -- Install DevkitPPC, Ninja and Python: - - **Only tested with DevkitPPC r41 and Python 3.8.10**, however other versions should work fine. -- Install Python modules from requirements.txt (`pip install -r requirements.txt`) -- Run configure.py -- Run ninja - -## Credits - -- jamchamb, Cuyler36, NWPlayer123 and fraser125 for past documentation of Animal Crossing. -- SeekyCt for [ppcdis](https://github.com/SeekyCt/ppcdis/) and helping setting up the project. -- msg for helping with *tools/map.py*. +# Animal Crossing Decompilation + +Decompilation in progress of Animal Crossing (GAFE01) + +## Cloning + +Use `--recursive` when cloning to have ppcdis in the repository. + +## Build + +**Currently, it has only been tested with [wibo](https://github.com/decompals/wibo) under WSL2, but should build under other platforms and with Wine.** + +### Building + +- Dump a copy of the game and extract **main.dol** and **foresta.rel.szs**. +- Decompress **foresta.rel.szs** with yaz0 found in *tools/*. +- Place **main.dol** and **foresta.rel** in *dump/*. +- Place CodeWarrior 1.3.2 in *tools/1.3.2/*, 1.2.5 in *tools/1.2.5/* and 1.2.5e in *tools/1.2.5e/*. +- Install DevkitPPC, Ninja and Python: + - **Only tested with DevkitPPC r41 and Python 3.8.10**, however other versions should work fine. +- Install Python modules from requirements.txt (`pip install -r requirements.txt`) +- Run configure.py +- Run ninja + +## Credits + +- jamchamb, Cuyler36, NWPlayer123 and fraser125 for past documentation of Animal Crossing. +- SeekyCt for [ppcdis](https://github.com/SeekyCt/ppcdis/) and helping setting up the project. +- msg for helping with *tools/map.py*. diff --git a/common.py b/common.py index a0d180af..c186597f 100644 --- a/common.py +++ b/common.py @@ -1,412 +1,413 @@ -""" -Common functions & definitions -""" - -from dataclasses import dataclass -from enum import Enum -from hashlib import sha1 -import json -import os -from subprocess import PIPE, run -from sys import executable as PYTHON, platform -from typing import List, Tuple, Union - -import yaml -try: - from yaml import CLoader as Loader -except ImportError: - from yaml import Loader - -############# -# Functions # -############# - -def get_file_sha1(path: str) -> bytes: - """Gets the SHA1 hash of a file""" - - with open(path, 'rb') as f: - return sha1(f.read()).digest() - -def get_cmd_stdout(cmd: str, text=True) -> str: - """Run a command and get the stdout output as a string""" - - ret = run(cmd.split(), stdout=PIPE, text=text) - assert ret.returncode == 0, f"Command '{cmd}' returned {ret.returncode}" - return ret.stdout - -class Binary(Enum): - DOL = 1 - REL = 2 - -# ppcdis source output -SourceDesc = Union[str, Tuple[str, int, int]] - -def get_containing_slice(addr: int) -> Tuple[Binary, SourceDesc]: - """Finds the binary containing an address and its source file - Source file is empty string if not decompiled""" - - dol_raw = get_cmd_stdout(f"{SLICES} {DOL_YML} {DOL_SLICES} -p {DOL_SRCDIR}/ --containing {addr:x}") - containing = json.loads(dol_raw) - if containing is None: - rel_raw = get_cmd_stdout( - f"{SLICES} {REL_YML} {REL_SLICES} -p {REL_SRCDIR}/ --containing {addr:x}" - ) - containing = json.loads(rel_raw) - assert containing is not None, f"Unknown address {addr:x}" - return (Binary.REL, containing) - else: - return (Binary.DOL, containing) - -def lookup_sym(sym: str, dol: bool = False, rel: bool = False, source_name: str = None) -> int: - """Takes a symbol as a name or address and returns the address""" - - # Get binary - if dol: - binary_name = DOL_YML - elif rel: - binary_name = REL_YML - else: - binary_name = None - - # Determine type - try: - return int(sym, 16) - except ValueError: - return get_address(sym, binary_name, source_name) - -def lookup_sym_full(sym: str, dol: bool = False, rel: bool = False, source_name: str = None - ) -> int: - """Takes a symbol as a name or address and returns both the name and address""" - - # Get binary - if dol: - binary_name = DOL_YML - elif rel: - binary_name = REL_YML - else: - binary_name = None - - # Determine type - try: - return int(sym, 16), get_name(sym) - except ValueError: - return get_address(sym, binary_name, source_name), sym - -def get_address(name: str, binary: bool = None, source_name: bool = None) -> int: - """Finds the address of a symbol""" - - args = [name] - if binary is not None: - args.append(f"-b {binary}") - if source_name is not None: - args.append(f"-n {source_name}") - - raw = get_cmd_stdout(f"{SYMBOLS} {GAME_SYMBOLS} --get-addr {' '.join(args)}") - return json.loads(raw) - -def get_name(addr: int, binary: bool = None, source_name: bool = None) -> int: - """Finds the name of a symbol""" - - args = [addr] - if binary is not None: - args.append(f"-b {binary}") - if source_name is not None: - args.append(f"-n {source_name}") - - raw = get_cmd_stdout(f"{SYMBOLS} {GAME_SYMBOLS} --get-name {' '.join(args)}") - return json.loads(raw) - -def find_headers(dirname: str, base=None) -> List[str]: - """Returns a list of all headers in a folder recursively""" - - if base is None: - base = dirname - - ret = [] - for name in os.listdir(dirname): - path = dirname + '/' + name - if os.path.isdir(path): - ret.extend(find_headers(path, base)) - elif name.endswith('.h'): - ret.append(path[len(base)+1:]) - - return ret - -def load_from_yaml(path: str, default=None): - """Loads an object from a yaml file""" - - if default is None: - default = {} - with open(path) as f: - ret = yaml.load(f.read(), Loader) - if ret is None: - ret = default - return ret - -################ -# Project dirs # -################ - -# Directory for decompiled dol code -DOL_SRCDIR = "src" - -# Directory for decompiled rel code -REL_SRCDIR = "rel" - -# Include directory -INCDIR = "include" - -# Build artifacts directory -BUILDDIR = "build" - -# Build include directory -BUILD_INCDIR = f"{BUILDDIR}/include" - -# Output binaries directory -OUTDIR = "out" - -# Original binaries directory -ORIG = "dump" - -# Tools directory -TOOLS = "tools" - -# Config directory -CONFIG = "config" - -# Extracted assets directory -ASSETS = "assets" - -######### -# Tools # -######### - -# ppcdis -PPCDIS = "tools/ppcdis" -PPCDIS_INCDIR = f"{PPCDIS}/include" -RELEXTERN = f"{PYTHON} {PPCDIS}/relextern.py" -ANALYSER = f"{PYTHON} {PPCDIS}/analyser.py" -DISASSEMBLER = f"{PYTHON} {PPCDIS}/disassembler.py" -ORDERSTRINGS = f"{PYTHON} {PPCDIS}/orderstrings.py" -ORDERFLOATS = f"{PYTHON} {PPCDIS}/orderfloats.py" -ASSETRIP = f"{PYTHON} {PPCDIS}/assetrip.py" -ASSETINC = f"{PYTHON} {PPCDIS}/assetinc.py" -FORCEACTIVEGEN = f"{PYTHON} {PPCDIS}/forceactivegen.py" -ELF2DOL = f"{PYTHON} {PPCDIS}/elf2dol.py" -ELF2REL = f"{PYTHON} {PPCDIS}/elf2rel.py" -SLICES = f"{PYTHON} {PPCDIS}/slices.py" -PROGRESS = f"{PYTHON} {PPCDIS}/progress.py" -SYMBOLS = f"{PYTHON} {PPCDIS}/symbols.py" -FORCEFILESGEN = f"{PYTHON} {PPCDIS}/forcefilesgen.py" - -# Codewarrior -TOOLS = "tools" -CODEWARRIOR = os.path.join(TOOLS, "1.3.2") -SDK_CW = os.path.join(TOOLS, "1.2.5") -HOTFIX_CW = os.path.join(TOOLS, "1.2.5e") -CC = os.path.join(CODEWARRIOR, "mwcceppc.exe") -OCC = os.path.join(SDK_CW, "mwcceppc.exe") -PROFILE = os.path.join(HOTFIX_CW, "mwcceppc.exe") -LD = os.path.join(CODEWARRIOR, "mwldeppc.exe") -if platform != "win32": - CC = f"wibo {CC}" - OCC = f"wibo {OCC}" - LD = f"wibo {LD}" - -# Frank -FRANKLITE = "tools/franklite.py" -FRANK = "tools/frank.py" - -# DevkitPPC -DEVKITPPC = os.environ.get("DEVKITPPC") -AS = os.path.join(DEVKITPPC, "bin", "powerpc-eabi-as") -OBJDUMP = os.path.join(DEVKITPPC, "bin", "powerpc-eabi-objdump") -CPP = os.path.join(DEVKITPPC, "bin", "powerpc-eabi-cpp") - -ICONV = f"{PYTHON} tools/sjis.py" # TODO: get actual iconv working(?) - -######### -# Files # -######### - -# Slices -DOL_SLICES = f"{CONFIG}/dol_slices.yml" -REL_SLICES = f"{CONFIG}/rel_slices.yml" - -# Overrides -ANALYSIS_OVERRIDES = f"{CONFIG}/analysis_overrides.yml" -DOL_DISASM_OVERRIDES = f"{CONFIG}/disasm_overrides.yml" -REL_DISASM_OVERRIDES = f"{CONFIG}/rel_disasm_overrides.yml" - -# Binaries -DOL = f"{ORIG}/main.dol" # read in python code -REL = f"{ORIG}/foresta.rel" # read in python code -DOL_YML = f"{CONFIG}/dol.yml" -REL_YML = f"{CONFIG}/rel.yml" -DOL_SHA = f"{ORIG}/main.dol.sha1" -REL_SHA = f"{ORIG}/foresta.rel.sha1" -DOL_OK = f"{BUILDDIR}/main.dol.ok" -REL_OK = f"{BUILDDIR}/foresta.rel.ok" -DOL_ASM_LIST = f"{BUILDDIR}/main.dol.asml" -REL_ASM_LIST = f"{BUILDDIR}/foresta.rel.asml" - -# Symbols -GAME_SYMBOLS = f"{CONFIG}/symbols.yml" - -# Assets -ASSETS_YML = f"{CONFIG}/assets.yml" - -# Analysis outputs -EXTERNS = f"{BUILDDIR}/externs.pickle" -DOL_LABELS = f"{BUILDDIR}/labels.pickle" -DOL_RELOCS = f"{BUILDDIR}/relocs.pickle" -REL_LABELS = f"{BUILDDIR}/rel_labels.pickle" -REL_RELOCS = f"{BUILDDIR}/rel_relocs.pickle" - -# Linker -DOL_LCF_TEMPLATE = f"{CONFIG}/dol.lcf" -DOL_LCF_TEMP = f"{BUILDDIR}/dol_temp.lcf" -DOL_LCF = f"{BUILDDIR}/dol.lcf" -REL_LCF = f"{CONFIG}/rel.lcf" - -# Outputs -DOL_ELF = f"{BUILDDIR}/main.elf" -REL_PLF = f"{BUILDDIR}/foresta.plf" -DOL_OUT = f"{OUTDIR}/main.dol" -REL_OUT = f"{OUTDIR}/foresta.rel" -DOL_MAP = f"{OUTDIR}/main.map" -REL_MAP = f"{OUTDIR}/foresta.map" - -# Optional full disassembly -DOL_FULL = f"{OUTDIR}/dol.s" -REL_FULL = f"{OUTDIR}/rel.s" - -############## -# Tool Flags # -############## - -ASFLAGS = ' '.join([ - "-m gekko", - f"-I {INCDIR}", - f"-I {PPCDIS_INCDIR}", - f"-I orig" -]) - -CPPFLAGS = ' '.join([ - "-nostdinc", - f"-I {INCDIR}", - f"-I {PPCDIS_INCDIR}", - f"-I {BUILD_INCDIR}" -]) - -DOL_SDATA2_SIZE = 8 -REL_SDATA2_SIZE = 0 - -CFLAGS = [ - "-O4", - "-char unsigned", - "-fp hard" -] -CPLFLAGS =[ - "-lang=c++", - "-O0" -] -BASE_DOL_CFLAGS = CFLAGS + [ - "-inline on", - "-sdata 8", - f"-sdata2 {DOL_SDATA2_SIZE}" -] -BASE_REL_CFLAGS = CFLAGS + [ - "-sdata 0", - f"-sdata2 {REL_SDATA2_SIZE}", - "-pool off" -] - -LOCAL_CFLAGS = [ - "-nostdinc", - "-proc gekko", - "-maxerrors 1", - "-Cpp_exceptions off", - "-I-", - f"-i {INCDIR}", - f"-i {PPCDIS_INCDIR}", - f"-i {BUILD_INCDIR}" -] - -PREPROCESSOR_CFLAGS = [ - "-E", - "-P" -] - -SDK_CFLAG = [ - "-O4,p", - "-inline all", - "-sdata 8", - f"-sdata2 {DOL_SDATA2_SIZE}" -] -ALIGN16_CFLAG = [ - "-func_align 16", -] -JSYSTEM_BASE = [ - "-lang=c++", - "-inline on", - "-fp fmadd", - "-fp_contract on", - #"-pool off", # this is wrong - "-Cpp_exceptions off", - "-RTTI on", - "-char signed", - "-enum int", - "-sym on", # might also be on for base flags? - "-O4,s" # in mkdd some libraries use O4,p, might be the case here too -] - -JSYSTEM_CFLAGS = ' '.join(JSYSTEM_BASE + LOCAL_CFLAGS) -DOL_CFLAGS = ' '.join(BASE_DOL_CFLAGS + LOCAL_CFLAGS) -SDK_FLAGS = ' '.join(SDK_CFLAG + LOCAL_CFLAGS) -ALIGN16 = ' '.join(BASE_DOL_CFLAGS + LOCAL_CFLAGS + ALIGN16_CFLAG) -DOL_CPPFLAGS = ' '.join(CPLFLAGS + BASE_DOL_CFLAGS + LOCAL_CFLAGS) -REL_CFLAGS = ' '.join(BASE_REL_CFLAGS + LOCAL_CFLAGS) -EXTERNAL_DOL_CFLAGS = ' '.join(BASE_DOL_CFLAGS) -EXTERNAL_REL_CFLAGS = ' '.join(BASE_REL_CFLAGS) -PREPROCESS_CFLAGS = ' '.join(PREPROCESSOR_CFLAGS) - -LDFLAGS = ' '.join([ - "-maxerrors 1", - "-mapunused" -]) - -PPCDIS_ANALYSIS_FLAGS = ' '.join([ - f"-o {ANALYSIS_OVERRIDES}", - f"-l {EXTERNS}" -]) - - -############ -# Contexts # -############ - -@dataclass -class SourceContext: - srcdir: str - cflags: str - binary: str - labels: str - relocs: str - slices: str - compiler: str - sdata2_threshold: int - disasm_overrides: int - -DOL_CTX = SourceContext(DOL_SRCDIR, DOL_CFLAGS, DOL_YML, DOL_LABELS, DOL_RELOCS, DOL_SLICES, - CC, DOL_SDATA2_SIZE, DOL_DISASM_OVERRIDES) -REL_CTX = SourceContext(REL_SRCDIR, REL_CFLAGS, REL_YML, REL_LABELS, REL_RELOCS, REL_SLICES, - CC, REL_SDATA2_SIZE, REL_DISASM_OVERRIDES) - -#################### -# diff.py Expected # -#################### - -EXPECTED = "expected" -DOL_EXPECTED = f"{EXPECTED}/build/main.elf" -REL_EXPECTED = f"{EXPECTED}/build/foresta.plf" +""" +Common functions & definitions +""" + +from dataclasses import dataclass +from enum import Enum +from hashlib import sha1 +import json +import os +from subprocess import PIPE, run +from sys import executable as PYTHON, platform +from typing import List, Tuple, Union + +import yaml +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + +############# +# Functions # +############# + +def get_file_sha1(path: str) -> bytes: + """Gets the SHA1 hash of a file""" + + with open(path, 'rb') as f: + return sha1(f.read()).digest() + +def get_cmd_stdout(cmd: str, text=True) -> str: + """Run a command and get the stdout output as a string""" + + ret = run(cmd.split(), stdout=PIPE, text=text) + assert ret.returncode == 0, f"Command '{cmd}' returned {ret.returncode}" + return ret.stdout + +class Binary(Enum): + DOL = 1 + REL = 2 + +# ppcdis source output +SourceDesc = Union[str, Tuple[str, int, int]] + +def get_containing_slice(addr: int) -> Tuple[Binary, SourceDesc]: + """Finds the binary containing an address and its source file + Source file is empty string if not decompiled""" + + dol_raw = get_cmd_stdout(f"{SLICES} {DOL_YML} {DOL_SLICES} -p {DOL_SRCDIR}/ --containing {addr:x}") + containing = json.loads(dol_raw) + if containing is None: + rel_raw = get_cmd_stdout( + f"{SLICES} {REL_YML} {REL_SLICES} -p {REL_SRCDIR}/ --containing {addr:x}" + ) + containing = json.loads(rel_raw) + assert containing is not None, f"Unknown address {addr:x}" + return (Binary.REL, containing) + else: + return (Binary.DOL, containing) + +def lookup_sym(sym: str, dol: bool = False, rel: bool = False, source_name: str = None) -> int: + """Takes a symbol as a name or address and returns the address""" + + # Get binary + if dol: + binary_name = DOL_YML + elif rel: + binary_name = REL_YML + else: + binary_name = None + + # Determine type + try: + return int(sym, 16) + except ValueError: + return get_address(sym, binary_name, source_name) + +def lookup_sym_full(sym: str, dol: bool = False, rel: bool = False, source_name: str = None + ) -> int: + """Takes a symbol as a name or address and returns both the name and address""" + + # Get binary + if dol: + binary_name = DOL_YML + elif rel: + binary_name = REL_YML + else: + binary_name = None + + # Determine type + try: + return int(sym, 16), get_name(sym) + except ValueError: + return get_address(sym, binary_name, source_name), sym + +def get_address(name: str, binary: bool = None, source_name: bool = None) -> int: + """Finds the address of a symbol""" + + args = [name] + if binary is not None: + args.append(f"-b {binary}") + if source_name is not None: + args.append(f"-n {source_name}") + + raw = get_cmd_stdout(f"{SYMBOLS} {GAME_SYMBOLS} --get-addr {' '.join(args)}") + return json.loads(raw) + +def get_name(addr: int, binary: bool = None, source_name: bool = None) -> int: + """Finds the name of a symbol""" + + args = [addr] + if binary is not None: + args.append(f"-b {binary}") + if source_name is not None: + args.append(f"-n {source_name}") + + raw = get_cmd_stdout(f"{SYMBOLS} {GAME_SYMBOLS} --get-name {' '.join(args)}") + return json.loads(raw) + +def find_headers(dirname: str, base=None) -> List[str]: + """Returns a list of all headers in a folder recursively""" + + if base is None: + base = dirname + + ret = [] + for name in os.listdir(dirname): + path = dirname + '/' + name + if os.path.isdir(path): + ret.extend(find_headers(path, base)) + elif name.endswith('.h'): + ret.append(path[len(base)+1:]) + + return ret + +def load_from_yaml(path: str, default=None): + """Loads an object from a yaml file""" + + if default is None: + default = {} + with open(path) as f: + ret = yaml.load(f.read(), Loader) + if ret is None: + ret = default + return ret + +################ +# Project dirs # +################ + +# Directory for decompiled dol code +DOL_SRCDIR = "src" + +# Directory for decompiled rel code +REL_SRCDIR = "rel" + +# Include directory +INCDIR = "include" + +# Build artifacts directory +BUILDDIR = "build" + +# Build include directory +BUILD_INCDIR = f"{BUILDDIR}/include" + +# Output binaries directory +OUTDIR = "out" + +# Original binaries directory +ORIG = "dump" + +# Tools directory +TOOLS = "tools" + +# Config directory +CONFIG = "config" + +# Extracted assets directory +ASSETS = "assets" + +######### +# Tools # +######### + +# ppcdis +PPCDIS = "tools/ppcdis" +PPCDIS_INCDIR = f"{PPCDIS}/include" +RELEXTERN = f"{PYTHON} {PPCDIS}/relextern.py" +ANALYSER = f"{PYTHON} {PPCDIS}/analyser.py" +DISASSEMBLER = f"{PYTHON} {PPCDIS}/disassembler.py" +ORDERSTRINGS = f"{PYTHON} {PPCDIS}/orderstrings.py" +ORDERFLOATS = f"{PYTHON} {PPCDIS}/orderfloats.py" +ASSETRIP = f"{PYTHON} {PPCDIS}/assetrip.py" +ASSETINC = f"{PYTHON} {PPCDIS}/assetinc.py" +FORCEACTIVEGEN = f"{PYTHON} {PPCDIS}/forceactivegen.py" +ELF2DOL = f"{PYTHON} {PPCDIS}/elf2dol.py" +ELF2REL = f"{PYTHON} {PPCDIS}/elf2rel.py" +SLICES = f"{PYTHON} {PPCDIS}/slices.py" +PROGRESS = f"{PYTHON} {PPCDIS}/progress.py" +SYMBOLS = f"{PYTHON} {PPCDIS}/symbols.py" +FORCEFILESGEN = f"{PYTHON} {PPCDIS}/forcefilesgen.py" + +# Codewarrior +TOOLS = "tools" +CODEWARRIOR = os.path.join(TOOLS, "1.3.2") +SDK_CW = os.path.join(TOOLS, "1.2.5") +HOTFIX_CW = os.path.join(TOOLS, "1.2.5e") +CC = os.path.join(CODEWARRIOR, "mwcceppc.exe") +OCC = os.path.join(SDK_CW, "mwcceppc.exe") +PROFILE = os.path.join(HOTFIX_CW, "mwcceppc.exe") +LD = os.path.join(CODEWARRIOR, "mwldeppc.exe") +if platform != "win32": + CC = f"wibo {CC}" + OCC = f"wibo {OCC}" + PROFILE = f"wibo {PROFILE}" + LD = f"wibo {LD}" + +# Frank +FRANKLITE = "tools/franklite.py" +FRANK = "tools/frank.py" + +# DevkitPPC +DEVKITPPC = os.environ.get("DEVKITPPC") +AS = os.path.join(DEVKITPPC, "bin", "powerpc-eabi-as") +OBJDUMP = os.path.join(DEVKITPPC, "bin", "powerpc-eabi-objdump") +CPP = os.path.join(DEVKITPPC, "bin", "powerpc-eabi-cpp") + +ICONV = f"{PYTHON} tools/sjis.py" # TODO: get actual iconv working(?) + +######### +# Files # +######### + +# Slices +DOL_SLICES = f"{CONFIG}/dol_slices.yml" +REL_SLICES = f"{CONFIG}/rel_slices.yml" + +# Overrides +ANALYSIS_OVERRIDES = f"{CONFIG}/analysis_overrides.yml" +DOL_DISASM_OVERRIDES = f"{CONFIG}/disasm_overrides.yml" +REL_DISASM_OVERRIDES = f"{CONFIG}/rel_disasm_overrides.yml" + +# Binaries +DOL = f"{ORIG}/main.dol" # read in python code +REL = f"{ORIG}/foresta.rel" # read in python code +DOL_YML = f"{CONFIG}/dol.yml" +REL_YML = f"{CONFIG}/rel.yml" +DOL_SHA = f"{ORIG}/main.dol.sha1" +REL_SHA = f"{ORIG}/foresta.rel.sha1" +DOL_OK = f"{BUILDDIR}/main.dol.ok" +REL_OK = f"{BUILDDIR}/foresta.rel.ok" +DOL_ASM_LIST = f"{BUILDDIR}/main.dol.asml" +REL_ASM_LIST = f"{BUILDDIR}/foresta.rel.asml" + +# Symbols +GAME_SYMBOLS = f"{CONFIG}/symbols.yml" + +# Assets +ASSETS_YML = f"{CONFIG}/assets.yml" + +# Analysis outputs +EXTERNS = f"{BUILDDIR}/externs.pickle" +DOL_LABELS = f"{BUILDDIR}/labels.pickle" +DOL_RELOCS = f"{BUILDDIR}/relocs.pickle" +REL_LABELS = f"{BUILDDIR}/rel_labels.pickle" +REL_RELOCS = f"{BUILDDIR}/rel_relocs.pickle" + +# Linker +DOL_LCF_TEMPLATE = f"{CONFIG}/dol.lcf" +DOL_LCF_TEMP = f"{BUILDDIR}/dol_temp.lcf" +DOL_LCF = f"{BUILDDIR}/dol.lcf" +REL_LCF = f"{CONFIG}/rel.lcf" + +# Outputs +DOL_ELF = f"{BUILDDIR}/main.elf" +REL_PLF = f"{BUILDDIR}/foresta.plf" +DOL_OUT = f"{OUTDIR}/main.dol" +REL_OUT = f"{OUTDIR}/foresta.rel" +DOL_MAP = f"{OUTDIR}/main.map" +REL_MAP = f"{OUTDIR}/foresta.map" + +# Optional full disassembly +DOL_FULL = f"{OUTDIR}/dol.s" +REL_FULL = f"{OUTDIR}/rel.s" + +############## +# Tool Flags # +############## + +ASFLAGS = ' '.join([ + "-m gekko", + f"-I {INCDIR}", + f"-I {PPCDIS_INCDIR}", + f"-I orig" +]) + +CPPFLAGS = ' '.join([ + "-nostdinc", + f"-I {INCDIR}", + f"-I {PPCDIS_INCDIR}", + f"-I {BUILD_INCDIR}" +]) + +DOL_SDATA2_SIZE = 8 +REL_SDATA2_SIZE = 0 + +CFLAGS = [ + "-O4", + "-char unsigned", + "-fp hard" +] +CPLFLAGS =[ + "-lang=c++", + "-O0" +] +BASE_DOL_CFLAGS = CFLAGS + [ + "-inline on", + "-sdata 8", + f"-sdata2 {DOL_SDATA2_SIZE}" +] +BASE_REL_CFLAGS = CFLAGS + [ + "-sdata 0", + f"-sdata2 {REL_SDATA2_SIZE}", + "-pool off" +] + +LOCAL_CFLAGS = [ + "-nostdinc", + "-proc gekko", + "-maxerrors 1", + "-Cpp_exceptions off", + "-I-", + f"-i {INCDIR}", + f"-i {PPCDIS_INCDIR}", + f"-i {BUILD_INCDIR}" +] + +PREPROCESSOR_CFLAGS = [ + "-E", + "-P" +] + +SDK_CFLAG = [ + "-O4,p", + "-inline all", + "-sdata 8", + f"-sdata2 {DOL_SDATA2_SIZE}" +] +ALIGN16_CFLAG = [ + "-func_align 16", +] +JSYSTEM_BASE = [ + "-lang=c++", + "-inline on", + "-fp fmadd", + "-fp_contract on", + #"-pool off", # this is wrong + "-Cpp_exceptions off", + "-RTTI on", + "-char signed", + "-enum int", + "-sym on", # might also be on for base flags? + "-O4,s" # in mkdd some libraries use O4,p, might be the case here too +] + +JSYSTEM_CFLAGS = ' '.join(JSYSTEM_BASE + LOCAL_CFLAGS) +DOL_CFLAGS = ' '.join(BASE_DOL_CFLAGS + LOCAL_CFLAGS) +SDK_FLAGS = ' '.join(SDK_CFLAG + LOCAL_CFLAGS) +ALIGN16 = ' '.join(BASE_DOL_CFLAGS + LOCAL_CFLAGS + ALIGN16_CFLAG) +DOL_CPPFLAGS = ' '.join(CPLFLAGS + BASE_DOL_CFLAGS + LOCAL_CFLAGS) +REL_CFLAGS = ' '.join(BASE_REL_CFLAGS + LOCAL_CFLAGS) +EXTERNAL_DOL_CFLAGS = ' '.join(BASE_DOL_CFLAGS) +EXTERNAL_REL_CFLAGS = ' '.join(BASE_REL_CFLAGS) +PREPROCESS_CFLAGS = ' '.join(PREPROCESSOR_CFLAGS) + +LDFLAGS = ' '.join([ + "-maxerrors 1", + "-mapunused" +]) + +PPCDIS_ANALYSIS_FLAGS = ' '.join([ + f"-o {ANALYSIS_OVERRIDES}", + f"-l {EXTERNS}" +]) + + +############ +# Contexts # +############ + +@dataclass +class SourceContext: + srcdir: str + cflags: str + binary: str + labels: str + relocs: str + slices: str + compiler: str + sdata2_threshold: int + disasm_overrides: int + +DOL_CTX = SourceContext(DOL_SRCDIR, DOL_CFLAGS, DOL_YML, DOL_LABELS, DOL_RELOCS, DOL_SLICES, + CC, DOL_SDATA2_SIZE, DOL_DISASM_OVERRIDES) +REL_CTX = SourceContext(REL_SRCDIR, REL_CFLAGS, REL_YML, REL_LABELS, REL_RELOCS, REL_SLICES, + CC, REL_SDATA2_SIZE, REL_DISASM_OVERRIDES) + +#################### +# diff.py Expected # +#################### + +EXPECTED = "expected" +DOL_EXPECTED = f"{EXPECTED}/build/main.elf" +REL_EXPECTED = f"{EXPECTED}/build/foresta.plf" diff --git a/configure.py b/configure.py index 634c7c56..cb199184 100644 --- a/configure.py +++ b/configure.py @@ -1,896 +1,875 @@ -""" -Creates a build script for ninja -""" -from abc import ABC, abstractmethod -from collections import defaultdict -from dataclasses import dataclass - -import json -import os -import pickle -import re -from io import StringIO - -from sys import executable as PYTHON, platform -from typing import List, Tuple -from ninja_syntax import Writer - -import common as c - -#################### -# Setup Validation # -#################### - -# Check CodeWarrior was added -assert os.path.exists("tools/1.3.2/mwcceppc.exe") and \ - os.path.exists("tools/1.2.5/mwcceppc.exe") and \ - os.path.exists("tools/1.3.2/mwldeppc.exe"), \ - "Error: Codewarrior not found!" - -# Check binaries were added -assert os.path.exists(c.DOL) and os.path.exists(c.REL), \ - "Error: Base binaries not found!" - -#Check if binaries are correct -dol_hash = c.get_file_sha1(c.DOL) -rel_hash = c.get_file_sha1(c.REL) - -assert dol_hash == bytes.fromhex("2ae8f56e7791d37e165bd5900921f2269f9515bf") and rel_hash == bytes.fromhex("c59d278ad8542bb05d6cbb632f60a0db05bef203"), \ - "Error: Base binaries hashes aren't correct!" - -#Check submodules added -assert os.path.exists(c.PPCDIS), \ - "Error: Git submodules not initialised!" - -############### -# Ninja Setup # -############### - -outbuf = StringIO() -n = Writer(outbuf) -n.variable("ninja_required_version", "1.3") -n.newline() - -################ -# Project Dirs # -################ - -n.variable("builddir", c.BUILDDIR) -n.variable("outdir", c.OUTDIR) -n.variable("dump", c.ORIG) -n.variable("tools", c.TOOLS) -n.variable("config", c.CONFIG) -n.newline() - -os.makedirs(c.BUILDDIR, exist_ok=True) - -######### -# Tools # -######### - -n.variable("python", c.PYTHON) -n.variable("ppcdis", c.PPCDIS) -n.variable("relextern", c.RELEXTERN) -n.variable("analyser", c.ANALYSER) -n.variable("disassembler", c.DISASSEMBLER) -n.variable("orderstrings", c.ORDERSTRINGS) -n.variable("orderfloats", c.ORDERFLOATS) -n.variable("assetrip", c.ASSETRIP) -n.variable("assetinc", c.ASSETINC) -n.variable("forceactivegen", c.FORCEACTIVEGEN) -n.variable("elf2dol", c.ELF2DOL) -n.variable("elf2rel", c.ELF2REL) -n.variable("codewarrior", c.CODEWARRIOR) -n.variable("cc", c.CC) -n.variable("franklite", c.FRANKLITE) -n.variable("frank", c.FRANK) -n.variable("occ", c.OCC) -n.variable("align16", c.ALIGN16) -n.variable("ld", c.LD) -n.variable("devkitppc", c.DEVKITPPC) -n.variable("as", c.AS) -n.variable("cpp", c.CPP) -n.variable("iconv", c.ICONV) -n.variable("forcefilesgen", c.FORCEFILESGEN) -n.newline() - -############## -# Tool Flags # -############## - -n.variable("asflags", c.ASFLAGS) -n.variable("ldflags", c.LDFLAGS) -n.variable("cppflags", c.CPPFLAGS) -n.variable("cplflags", c.CPLFLAGS) -n.variable("ppcdis_analysis_flags", c.PPCDIS_ANALYSIS_FLAGS) -n.newline() - -######### -# Rules # -######### - -# Windows can't use && without this statement -ALLOW_CHAIN = "cmd /c " if os.name == "nt" else "" -mwcc_cmd = ALLOW_CHAIN + f"$cpp -M $in -MF $out.d $cppflags && $cc $cflags -c $in -o $out" - -n.rule( - "relextern", - command = "$relextern $out $in", - description = "ppcdis rel extern $in" -) - -n.rule( - "analyse", - command = "$analyser $in $out $analysisflags", - description = "ppcdis analysis $in", - pool="console" -) - -n.rule( - "disasm", - command = "$disassembler $in $out -q $disasmflags", - description = "ppcdis full disassembly $out" -) - -n.rule( - "disasm_slice", - command = "$disassembler $in $out -q $disasmflags -s $slice", - description = "ppcdis disassembly $out" -) - -n.rule( - "disasm_single", - command = "$disassembler $in $out -j $addr -q $disasmflags", - description = "ppcdis function disassembly $addr" -) - -n.rule( - "jumptable", - command = "$disassembler $in $out -j $addr -q $disassembler", - description = "Jumptable $addr" -) - -n.rule( - "orderstrings", - command = "$orderstrings $in $addrs $out $flags --enc shift-jis --pool", - description = "Order strings $in $addrs" -) - -n.rule( - "orderfloats", - command = "$orderfloats $in $addrs $out $flags", - description = "Order floats $in $addrs" -) - -n.rule( - "assetrip", - command = "$assetrip $in $addrs $out", - description = "Asset rip $out" -) - -n.rule( - "assetinc", - command = "$assetinc $in $out", - description = "Asset include generation $out" -) - -n.rule( - "forceactivegen", - command = "$forceactivegen $in $out", - description = "LCF FORCEACTIVEGEN generation $in" -) - -n.rule( - "elf2dol", - command = "$elf2dol $in -o $out", - description = "elf2dol $in" -) - -n.rule( - "elf2rel", - command = "$elf2rel $in -o $out $flags", - description = "elf2rel $in" -) - -n.rule( - "sha1sum", - command = ALLOW_CHAIN + "sha1sum -c $in && touch $out", - description ="Verify $in", - pool ="console" -) - -n.rule( - "as", - command = f"$as $asflags -c $in -o $out", - description = "AS $in" -) - -n.rule( - "cc", - command = mwcc_cmd, - description = "CC $in", - deps = "gcc", - depfile = "$out.d" -) - -n.rule( - "franklite", - command = f"{mwcc_cmd} && $python $franklite $basefile $basefile", - description = "FRANKLITE $out", - deps = "gcc", - depfile = "$basefile.d" -) - -n.rule( - "frank", - command = f"{mwcc_cmd} " + - f"&& {c.PROFILE} $cflags -c $in -o $out.profile " + - f"&& $python $frank $out $out.profile $out ", - description = "FRANK $out", - deps = "gcc", - depfile = "$out.d" -) - -n.rule( - "ccs", - command = ALLOW_CHAIN + f"$cpp -M $in -MF $out.d $cppflags && $cc $cflags -S $in -o $out", - description = "CC -S $in", - deps = "gcc", - depfile = "$out.d" -) - -n.rule( - "ld", - command = "$ld $ldflags -map $map -lcf $lcf $in -o $out", - description = "LD $out", -) - -n.rule( - "iconv", - command = "$iconv $in $out", - description = "iconv $in", -) - -n.rule( - "forcefiles", - command = "$forcefilesgen $in $out $forcefiles", - description = "LCF FORCEFILES generation $in" -) - -########## -# Assets # -########## - -@dataclass -class Asset: - binary: str - path: str - start: int - end: int - - def load(yml_path: str): - return { - asset : Asset(binary, asset, *adat["addrs"]) - for binary, bdat in c.load_from_yaml(yml_path).items() - for asset, adat in bdat.items() - } - - def dump(self): - print(f"Ripping {self.path} from main.dol") - os.system( - f"{PYTHON} {c.PPCDIS}/assetrip.py {c.DOL_YML} 0x{self.start:x} {self.end:x} {c.ASSETS}/{self.path}" - ) - print(f"Converting {self.path} from main.dol") - os.system( - f"{PYTHON} {c.PPCDIS}/assetinc.py {c.ASSETS}/{self.path} {c.ASSETS}/{self.path}.inc" - ) -assets = Asset.load(c.ASSETS_YML) - - -############## -# Rip Assets # -############## - -for asset in assets.values(): - Asset.dump(asset) - - - -########### -# Sources # -########### - -class GeneratedInclude(ABC): - def __init__(self, ctx: c.SourceContext, source_name: str, path: str): - self.ctx = ctx - self.source_name = source_name - self.path = path - - @abstractmethod - def build(self): - raise NotImplementedError - - def find(ctx: c.SourceContext, source_name: str, txt: str) -> List["GeneratedInclude"]: - return [ - cl(ctx, source_name, match) - for cl in ( - AsmInclude, - JumptableInclude, - StringInclude, - FloatInclude, - DoubleInclude, - AssetInclude - ) - for match in re.findall(cl.REGEX, txt) - ] - -class AsmInclude(GeneratedInclude): - REGEX = r'#include "asm\/([0-9a-f]{8})\.s"' - - def __init__(self, ctx: c.SourceContext, source_name: str, match: str): - self.addr = match - super().__init__(ctx, source_name, f"{c.BUILD_INCDIR}/asm/{self.addr}.s") - - def build(includes: List["AsmInclude"]): - # Skip empty list - if len(includes) == 0: - return - - # Get ctx from first include (all should be equal) - ctx = includes[0].ctx - - # Sort by source name - batches = defaultdict(list) - for inc in includes: - batches[inc.source_name].append(inc) - - # Compile by source name - for source_name, incs in batches.items(): - n.build( - [inc.path for inc in incs], - rule="disasm_single", - inputs=[ctx.binary, ctx.labels, ctx.relocs], - implicit=[c.GAME_SYMBOLS, ctx.disasm_overrides], - variables={ - "disasmflags" : f"-m {c.GAME_SYMBOLS} -o {ctx.disasm_overrides} -n {source_name}", - "addr" : ' '.join(inc.addr for inc in incs) } - ) - - def _repr__(self): - return f"AsmInclude({self.addr})" - -class JumptableInclude(GeneratedInclude): - REGEX = r'#include "jumptable\/([0-9a-f]{8})\.inc"' - - def __init__(self, ctx: c.SourceContext, source_name: str, match: str): - self.addr = match - super().__init__(ctx, source_name, f"{c.BUILD_INCDIR}/jumptable/{self.addr}.inc") - - def build(includes: List["JumptableInclude"]): - # Skip empty list - if len(includes) == 0: - return - - # Get context from first include (all should be equal) - ctx = includes[0].ctx - - # Sort by source name - batches = defaultdict(list) - for inc in includes: - batches[inc.source_name].append(inc) - - # Compile by source name - # TODO: subdivide large batches - for source_name, incs in batches.items(): - n.build( - [inc.path for inc in incs], - rule="jumptable", - inputs=[ctx.binary, ctx.labels, ctx.relocs], - implicit=[c.GAME_SYMBOLS, ctx.disasm_overrides], - variables={ - "disasmflags" : f"-m {c.GAME_SYMBOLS} -o {ctx.disasm_overrides} -n {source_name}", - "addr" : ' '.join(inc.addr for inc in incs) - } - ) - - def __repr__(self): - return f"JumptableInclude({self.addr})" - - -class StringInclude(GeneratedInclude): - REGEX = r'#include "orderstrings\/([0-9a-f]{8})_([0-9a-f]{8})\.inc"' - - def __init__(self, ctx: c.SourceContext, source_name: str, match: Tuple[str]): - self.start, self.end = match - super().__init__(ctx, source_name, - f"{c.BUILD_INCDIR}/orderstrings/{self.start}_{self.end}.inc") - - def build(includes: List["StringInclude"]): - # Skip empty list - if len(includes) == 0: - return - - # Get context from first include (all should be equal) - ctx = includes[0].ctx - - # Build - for inc in includes: - n.build( - inc.path, - rule="orderstrings", - inputs=ctx.binary, - variables={ - "addrs" : f"{inc.start} {inc.end}" - } - ) - - def __repr__(self): - return f"StringInclude({self.start}, {self.end})" - - -class FloatInclude(GeneratedInclude): - REGEX = r'#include "(orderfloats(m?))\/([0-9a-f]{8})_([0-9a-f]{8})\.inc"' - - def __init__(self, ctx: c.SourceContext, source_name: str, match: Tuple[str]): - folder, manual, self.start, self.end = match - self.manual = manual != '' - super().__init__(ctx, source_name, - f"{c.BUILD_INCDIR}/{folder}/{self.start}_{self.end}.inc") - - def build(includes: List["FloatInclude"]): - # Skip empty list - if len(includes) == 0: - return - - # Get context from first include (all should be equal) - ctx = includes[0].ctx - - # Build - for inc in includes: - sda = "--sda " if ctx.sdata2_threshold >= 4 else "" - asm = "" if inc.manual else "--asm" - n.build( - inc.path, - rule="orderfloats", - inputs=inc.ctx.binary, - variables={ - "addrs" : f"{inc.start} {inc.end}", - "flags" : f"{sda} {asm}" - } - ) - - def __repr__(self): - return f"FloatInclude({self.start}, {self.end})" - -class DoubleInclude(GeneratedInclude): - REGEX = r'#include "orderdoubles\/([0-9a-f]{8})_([0-9a-f]{8})\.inc"' - - def __init__(self, ctx: c.SourceContext, source_name: str, match: Tuple[str]): - self.start, self.end = match - super().__init__(ctx, source_name, - f"{c.BUILD_INCDIR}/orderdoubles/{self.start}_{self.end}.inc") - - def build(includes: List["DoubleInclude"]): - # Skip empty list - if len(includes) == 0: - return - - # Get context from first include (all should be equal) - ctx = includes[0].ctx - - # Build - for inc in includes: - n.build( - inc.path, - rule="orderfloats", - inputs=ctx.binary, - variables={ - "addrs" : f"{inc.start} {inc.end}", - "flags" : f"--double" - } - ) - - def __repr__(self): - return f"DoubleInclude({self.start}, {self.end})" - -class AssetInclude(GeneratedInclude): - REGEX = r'#include "assets\/(.+)\.inc"' - - def __init__(self, ctx: c.SourceContext, source_name: str, match: Tuple[str]): - self.asset = assets[match] - self.asset_path = f"{c.ASSETS}/{self.asset.path}" - assert ctx.binary == self.asset.binary, f"Tried to include from other binary" - super().__init__(ctx, source_name, f"{c.BUILD_INCDIR}/assets/{self.asset.path}.inc") - - def build(includes: List["AssetInclude"]): - # Skip empty list - if len(includes) == 0: - return - - # Build - for inc in includes: - n.build( - inc.asset_path, - rule="assetrip", - inputs=inc.asset.binary, - variables={ - "addrs" : f"{inc.asset.start:x} {inc.asset.end:x}" - } - ) - n.build( - inc.path, - rule="assetinc", - inputs=inc.asset_path - ) - - def __repr__(self): - return f"AssetInclude({self.asset})" - -class Source(ABC): - def __init__(self, decompiled: bool, src_path: str, o_path: str, - gen_includes: List[GeneratedInclude] = []): - self.decompiled = decompiled - self.src_path = src_path - self.o_path = o_path - self.o_stem = o_path[:-2] - self.i_path = o_path[:-2] + ".i" - self.gen_includes = gen_includes - - def build(self): - raise NotImplementedError - - def make(ctx: c.SourceContext, source: c.SourceDesc): - if isinstance(source, str): - print(source) - ext = source.split('.')[-1].lower() - if ext in ("c", "cpp", "cxx", "cc"): - return CSource(ctx, source) - elif ext == "s": - return AsmSource(ctx, source) - else: - assert 0, f"Unknown source type .{ext}" - else: - return GenAsmSource(ctx, *source) - -class GenAsmSource(Source): - def __init__(self, ctx: c.SourceContext, section: str, start: int, end: int): - self.start = start - self.end = end - self.ctx = ctx - name = f"{section}_{start:x}_{end:x}.s" - src_path = f"$builddir/asm/{name}" - super().__init__(False, src_path, src_path + ".o") - - # Add ctors to forcefiles - if section == ".ctors": - forcefiles.append(name + ".o") - - - def build(self): - n.build( - self.src_path, - rule = "disasm_slice", - inputs = [self.ctx.binary, self.ctx.labels, self.ctx.relocs], - implicit = [c.GAME_SYMBOLS, self.ctx.disasm_overrides], - variables = { - "slice" : f"{self.start:x} {self.end:x}", - "disasmflags" : f"-m {c.GAME_SYMBOLS} -o {self.ctx.disasm_overrides}" - } - ) - n.build( - self.o_path, - rule="as", - inputs=self.src_path - ) - def batch_build(sources: List["GenAsmSource"], batch_size=20): - # TODO: configure batch size based on cpu core count - - # Skip empty list - if len(sources) == 0: - return - - # Get context from first include (all should be equal) - ctx = sources[0].ctx - - for src in sources: - n.build( - src.o_path, - rule="as", - inputs=src.src_path - ) - - while len(sources) > 0: - batch, sources = sources[:batch_size], sources[batch_size:] - n.build( - [src.src_path for src in batch], - rule = "disasm_slice", - inputs = [ctx.binary, ctx.labels, ctx.relocs], - implicit = [c.GAME_SYMBOLS, ctx.disasm_overrides], - variables = { - "slice" : ' '.join( - f"{src.start:x} {src.end:x}" - for src in batch - ), - "disasmflags" : f"-m {c.GAME_SYMBOLS} -o {ctx.disasm_overrides}" - } - ) - -class AsmSource(Source): - def __init__(self, ctx: c.SourceContext, path: str): - super().__init__(True, path, f"$builddir/{path}.o") - - def build(self): - n.build( - self.o_path, - rule = "as", - inputs = self.src_path - ) - -class CSource(Source): - def __init__(self, ctx: c.SourceContext, path: str): - if path.startswith("src/dolphin/"): - self.cflags = c.SDK_FLAGS - self.cc = c.OCC - self.frank = True - elif path.startswith("src/JSystem/"): - self.cflags = c.JSYSTEM_CFLAGS - self.cc = c.CC - self.frank = False - elif path.startswith("src/jaudio_NES"): - self.cc = c.CC - self.cflags = c.DOL_CPPFLAGS - self.frank = False - else: - self.cflags = ctx.cflags - self.cc = c.CC - self.frank = False - self.iconv_path = f"$builddir/iconv/{path}" - - # Find generated includes - with open(path, encoding="utf-8") as f: - gen_includes = GeneratedInclude.find(ctx, path, f.read()) - - self.s_path = f"$builddir/{path}.s" - super().__init__(True, path, f"$builddir/{path}.o", gen_includes) - - def build(self): - n.build( - self.iconv_path, - rule="iconv", - inputs=self.src_path - ) - #n.build( - # self.o_path, - # rule = "cc", - # inputs = self.iconv_path, - # implicit = [inc.path for inc in self.gen_includes], - # variables = { - # "cc" : self.cc, - # "cflags" : self.cflags + ' ' + c.PREPROCESS_CFLAGS - # } - #) - #print(self.i_path) - - if self.frank == True: - #print(f"python3 franklite.py {self.o_path} {self.o_path}") - n.build( - self.o_path, - rule = "frank", - inputs = self.iconv_path, - implicit = [inc.path for inc in self.gen_includes], - variables = { - "cc" : self.cc, - "cflags" : self.cflags, - #"basefile" : self.o_path - } - ) - else: - n.build( - self.o_path, - rule = "cc", - inputs = self.iconv_path, - implicit = [inc.path for inc in self.gen_includes], - variables = { - "cc" : self.cc, - "cflags" : self.cflags - } - ) - # Optional manual debug target - n.build( - self.s_path, - rule = "ccs", - inputs = self.iconv_path, - implicit = [inc.path for inc in self.gen_includes], - variables = { - "cflags" : self.cflags - } - ) - -def load_sources(ctx: c.SourceContext): - raw = c.get_cmd_stdout( - f"{c.SLICES} {ctx.binary} {ctx.slices} -o -p {ctx.srcdir}/" - ) - return [Source.make(ctx, s) for s in json.loads(raw)] - -def find_gen_includes(sources: List[Source]): - ret = defaultdict(list) - for source in sources: - if not isinstance(source, CSource): - continue - - for inc in source.gen_includes: - ret[type(inc)].append(inc) - - return ret - -def make_asm_list(path: str, asm_includes: List[AsmInclude]): - with open(path, 'wb') as f: - pickle.dump( - [ - int(inc.addr, 16) - for inc in asm_includes - ], - f - ) - -forcefiles = [] - -dol_sources = load_sources(c.DOL_CTX) -dol_gen_includes = find_gen_includes(dol_sources) -make_asm_list(c.DOL_ASM_LIST, dol_gen_includes[AsmInclude]) - -rel_sources = load_sources(c.REL_CTX) -rel_gen_includes = find_gen_includes(rel_sources) -make_asm_list(c.REL_ASM_LIST, rel_gen_includes[AsmInclude]) - -########## -# Builds # -########## - -n.build( - c.EXTERNS, - rule = "relextern", - inputs = c.REL_YML -) - -n.build( - [c.REL_LABELS, c.REL_RELOCS], - rule = "analyse", - inputs = c.REL_YML, - implicit = [c.ANALYSIS_OVERRIDES, c.EXTERNS], - variables = { - "analysisflags" : "$ppcdis_analysis_flags" - } -) - -n.build( - [c.DOL_LABELS, c.DOL_RELOCS], - rule = "analyse", - inputs = c.DOL_YML, - implicit = [c.ANALYSIS_OVERRIDES, c.EXTERNS], - variables = { - "analysisflags" : f"$ppcdis_analysis_flags" - } -) - -for cl, includes in dol_gen_includes.items(): - cl.build(includes) - -for cl, includes in rel_gen_includes.items(): - cl.build(includes) - -dol_gen_asm = [] -for source in dol_sources: - if isinstance(source, GenAsmSource): - dol_gen_asm.append(source) - else: - source.build() -GenAsmSource.batch_build(dol_gen_asm) - -rel_gen_asm = [] -for source in rel_sources: - if isinstance(source, GenAsmSource): - rel_gen_asm.append(source) - else: - source.build() -GenAsmSource.batch_build(rel_gen_asm) - -n.build( - c.DOL_LCF_TEMP, - rule="forceactivegen", - inputs=[c.DOL_LCF_TEMPLATE, c.DOL_YML, c.DOL_LABELS, c.GAME_SYMBOLS, c.EXTERNS] -) - -n.build( - c.DOL_LCF, - rule="forcefiles", - inputs=c.DOL_LCF_TEMP, - variables={ - "forcefiles" : ' '.join(forcefiles) - } -) - -n.build( - c.DOL_ELF, - rule="ld", - inputs=[s.o_path for s in dol_sources], - implicit=c.DOL_LCF, - implicit_outputs=c.DOL_MAP, - variables={ - "map" : c.DOL_MAP, - "lcf" : c.DOL_LCF - } -) - -n.build( - c.DOL_OUT, - rule="elf2dol", - inputs=c.DOL_ELF, -) - -n.build( - c.DOL_OK, - rule = "sha1sum", - inputs = c.DOL_SHA, - implicit = [c.DOL_OUT] -) -n.default(c.DOL_OK) - -n.build( - c.REL_PLF, - rule="ld", - inputs=[s.o_path for s in rel_sources], - implicit=c.REL_LCF, - implicit_outputs=c.REL_MAP, - variables={ - "map" : c.REL_MAP, - "lcf" : c.REL_LCF, - "ldflags" : c.LDFLAGS + " -r1 -m _prolog" - } -) - -n.build( - c.REL_OUT, - rule="elf2rel", - inputs=[c.REL_PLF, c.DOL_ELF], - variables={ - "flags" : f"-n 20 --name-size 0x38 -v 2 -r {c.REL_YML}" - } -) - -n.build( - c.REL_OK, - rule = "sha1sum", - inputs = c.REL_SHA, - implicit = [c.REL_OUT] -) -n.default(c.REL_OK) - -# Optional full binary disassembly -n.build( - c.DOL_FULL, - rule = "disasm", - inputs=[c.DOL_YML, c.DOL_LABELS, c.DOL_RELOCS], - implicit=[c.GAME_SYMBOLS, c.DOL_DISASM_OVERRIDES], - variables={ - "disasmflags" : f"-m {c.GAME_SYMBOLS} -o {c.REL_DISASM_OVERRIDES}" - } -) - -n.build( - c.REL_FULL, - rule = "disasm", - inputs=[c.REL_YML, c.REL_LABELS, c.REL_RELOCS], - implicit=[c.GAME_SYMBOLS, c.REL_DISASM_OVERRIDES], - variables={ - "disasmflags" : f"-m {c.GAME_SYMBOLS} -o {c.REL_DISASM_OVERRIDES}" - } -) - -########## -# Ouptut # -########## - -with open("build.ninja", 'w') as f: - f.write(outbuf.getvalue()) -n.close() \ No newline at end of file +""" +Creates a build script for ninja +""" +from abc import ABC, abstractmethod +from collections import defaultdict +from dataclasses import dataclass + +import json +import os +import pickle +import re +from io import StringIO + +from sys import executable as PYTHON, platform +from typing import List, Tuple +from ninja_syntax import Writer + +import common as c + +#################### +# Setup Validation # +#################### + +# Check CodeWarrior was added +assert os.path.exists("tools/1.3.2/mwcceppc.exe") and \ + os.path.exists("tools/1.2.5/mwcceppc.exe") and \ + os.path.exists("tools/1.3.2/mwldeppc.exe"), \ + "Error: Codewarrior not found!" + +# Check binaries were added +assert os.path.exists(c.DOL) and os.path.exists(c.REL), \ + "Error: Base binaries not found!" + +#Check if binaries are correct +dol_hash = c.get_file_sha1(c.DOL) +rel_hash = c.get_file_sha1(c.REL) + +assert dol_hash == bytes.fromhex("2ae8f56e7791d37e165bd5900921f2269f9515bf") and rel_hash == bytes.fromhex("c59d278ad8542bb05d6cbb632f60a0db05bef203"), \ + "Error: Base binaries hashes aren't correct!" + +#Check submodules added +assert os.path.exists(c.PPCDIS), \ + "Error: Git submodules not initialised!" + +############### +# Ninja Setup # +############### + +outbuf = StringIO() +n = Writer(outbuf) +n.variable("ninja_required_version", "1.3") +n.newline() + +################ +# Project Dirs # +################ + +n.variable("builddir", c.BUILDDIR) +n.variable("outdir", c.OUTDIR) +n.variable("dump", c.ORIG) +n.variable("tools", c.TOOLS) +n.variable("config", c.CONFIG) +n.newline() + +os.makedirs(c.BUILDDIR, exist_ok=True) + +######### +# Tools # +######### + +n.variable("python", c.PYTHON) +n.variable("ppcdis", c.PPCDIS) +n.variable("relextern", c.RELEXTERN) +n.variable("analyser", c.ANALYSER) +n.variable("disassembler", c.DISASSEMBLER) +n.variable("orderstrings", c.ORDERSTRINGS) +n.variable("orderfloats", c.ORDERFLOATS) +n.variable("assetrip", c.ASSETRIP) +n.variable("assetinc", c.ASSETINC) +n.variable("forceactivegen", c.FORCEACTIVEGEN) +n.variable("elf2dol", c.ELF2DOL) +n.variable("elf2rel", c.ELF2REL) +n.variable("codewarrior", c.CODEWARRIOR) +n.variable("cc", c.CC) +n.variable("franklite", c.FRANKLITE) +n.variable("frank", c.FRANK) +n.variable("occ", c.OCC) +n.variable("align16", c.ALIGN16) +n.variable("ld", c.LD) +n.variable("devkitppc", c.DEVKITPPC) +n.variable("as", c.AS) +n.variable("cpp", c.CPP) +n.variable("iconv", c.ICONV) +n.variable("forcefilesgen", c.FORCEFILESGEN) +n.newline() + +############## +# Tool Flags # +############## + +n.variable("asflags", c.ASFLAGS) +n.variable("ldflags", c.LDFLAGS) +n.variable("cppflags", c.CPPFLAGS) +n.variable("cplflags", c.CPLFLAGS) +n.variable("ppcdis_analysis_flags", c.PPCDIS_ANALYSIS_FLAGS) +n.newline() + +######### +# Rules # +######### + +# Windows can't use && without this statement +ALLOW_CHAIN = "cmd /c " if os.name == "nt" else "" +mwcc_cmd = ALLOW_CHAIN + f"$cpp -M $in -MF $out.d $cppflags && $cc $cflags -c $in -o $out" + +n.rule( + "relextern", + command = "$relextern $out $in", + description = "ppcdis rel extern $in" +) + +n.rule( + "analyse", + command = "$analyser $in $out $analysisflags", + description = "ppcdis analysis $in", + pool="console" +) + +n.rule( + "disasm", + command = "$disassembler $in $out -q $disasmflags", + description = "ppcdis full disassembly $out" +) + +n.rule( + "disasm_slice", + command = "$disassembler $in $out -q $disasmflags -s $slice", + description = "ppcdis disassembly $out" +) + +n.rule( + "disasm_single", + command = "$disassembler $in $out -j $addr -q $disasmflags", + description = "ppcdis function disassembly $addr" +) + +n.rule( + "jumptable", + command = "$disassembler $in $out -j $addr -q $disassembler", + description = "Jumptable $addr" +) + +n.rule( + "orderstrings", + command = "$orderstrings $in $addrs $out $flags --enc shift-jis --pool", + description = "Order strings $in $addrs" +) + +n.rule( + "orderfloats", + command = "$orderfloats $in $addrs $out $flags", + description = "Order floats $in $addrs" +) + +n.rule( + "assetrip", + command = "$assetrip $in $addrs $out", + description = "Asset rip $out" +) + +n.rule( + "assetinc", + command = "$assetinc $in $out", + description = "Asset include generation $out" +) + +n.rule( + "forceactivegen", + command = "$forceactivegen $in $out", + description = "LCF FORCEACTIVEGEN generation $in" +) + +n.rule( + "elf2dol", + command = "$elf2dol $in -o $out", + description = "elf2dol $in" +) + +n.rule( + "elf2rel", + command = "$elf2rel $in -o $out $flags", + description = "elf2rel $in" +) + +n.rule( + "sha1sum", + command = ALLOW_CHAIN + "sha1sum -c $in && touch $out", + description ="Verify $in", + pool ="console" +) + +n.rule( + "as", + command = f"$as $asflags -c $in -o $out", + description = "AS $in" +) + +n.rule( + "cc", + command = mwcc_cmd, + description = "CC $in", + deps = "gcc", + depfile = "$out.d" +) + +n.rule( + "franklite", + command = f"{mwcc_cmd} && $python $franklite $basefile $basefile", + description = "FRANKLITE $out", + deps = "gcc", + depfile = "$basefile.d" +) + +n.rule( + "frank", + command = f"{mwcc_cmd} " + + f"&& {c.PROFILE} $cflags -c $in -o $out.profile " + + f"&& $python $frank $out $out.profile $out ", + description = "FRANK $out", + deps = "gcc", + depfile = "$out.d" +) + +n.rule( + "ccs", + command = ALLOW_CHAIN + f"$cpp -M $in -MF $out.d $cppflags && $cc $cflags -S $in -o $out", + description = "CC -S $in", + deps = "gcc", + depfile = "$out.d" +) + +n.rule( + "ld", + command = "$ld $ldflags -map $map -lcf $lcf $in -o $out", + description = "LD $out", +) + +n.rule( + "iconv", + command = "$iconv $in $out", + description = "iconv $in", +) + +n.rule( + "forcefiles", + command = "$forcefilesgen $in $out $forcefiles", + description = "LCF FORCEFILES generation $in" +) + +########## +# Assets # +########## + +@dataclass +class Asset: + binary: str + path: str + start: int + end: int + + def load(yml_path: str): + return { + asset : Asset(binary, asset, *adat["addrs"]) + for binary, bdat in c.load_from_yaml(yml_path).items() + for asset, adat in bdat.items() + } +assets = Asset.load(c.ASSETS_YML) +########### +# Sources # +########### + +class GeneratedInclude(ABC): + def __init__(self, ctx: c.SourceContext, source_name: str, path: str): + self.ctx = ctx + self.source_name = source_name + self.path = path + + @abstractmethod + def build(self): + raise NotImplementedError + + def find(ctx: c.SourceContext, source_name: str, txt: str) -> List["GeneratedInclude"]: + return [ + cl(ctx, source_name, match) + for cl in ( + AsmInclude, + JumptableInclude, + StringInclude, + FloatInclude, + DoubleInclude, + AssetInclude + ) + for match in re.findall(cl.REGEX, txt) + ] + +class AsmInclude(GeneratedInclude): + REGEX = r'#include "asm\/([0-9a-f]{8})\.s"' + + def __init__(self, ctx: c.SourceContext, source_name: str, match: str): + self.addr = match + super().__init__(ctx, source_name, f"{c.BUILD_INCDIR}/asm/{self.addr}.s") + + def build(includes: List["AsmInclude"]): + # Skip empty list + if len(includes) == 0: + return + + # Get ctx from first include (all should be equal) + ctx = includes[0].ctx + + # Sort by source name + batches = defaultdict(list) + for inc in includes: + batches[inc.source_name].append(inc) + + # Compile by source name + for source_name, incs in batches.items(): + n.build( + [inc.path for inc in incs], + rule="disasm_single", + inputs=[ctx.binary, ctx.labels, ctx.relocs], + implicit=[c.GAME_SYMBOLS, ctx.disasm_overrides], + variables={ + "disasmflags" : f"-m {c.GAME_SYMBOLS} -o {ctx.disasm_overrides} -n {source_name}", + "addr" : ' '.join(inc.addr for inc in incs) } + ) + + def _repr__(self): + return f"AsmInclude({self.addr})" + +class JumptableInclude(GeneratedInclude): + REGEX = r'#include "jumptable\/([0-9a-f]{8})\.inc"' + + def __init__(self, ctx: c.SourceContext, source_name: str, match: str): + self.addr = match + super().__init__(ctx, source_name, f"{c.BUILD_INCDIR}/jumptable/{self.addr}.inc") + + def build(includes: List["JumptableInclude"]): + # Skip empty list + if len(includes) == 0: + return + + # Get context from first include (all should be equal) + ctx = includes[0].ctx + + # Sort by source name + batches = defaultdict(list) + for inc in includes: + batches[inc.source_name].append(inc) + + # Compile by source name + # TODO: subdivide large batches + for source_name, incs in batches.items(): + n.build( + [inc.path for inc in incs], + rule="jumptable", + inputs=[ctx.binary, ctx.labels, ctx.relocs], + implicit=[c.GAME_SYMBOLS, ctx.disasm_overrides], + variables={ + "disasmflags" : f"-m {c.GAME_SYMBOLS} -o {ctx.disasm_overrides} -n {source_name}", + "addr" : ' '.join(inc.addr for inc in incs) + } + ) + + def __repr__(self): + return f"JumptableInclude({self.addr})" + + +class StringInclude(GeneratedInclude): + REGEX = r'#include "orderstrings\/([0-9a-f]{8})_([0-9a-f]{8})\.inc"' + + def __init__(self, ctx: c.SourceContext, source_name: str, match: Tuple[str]): + self.start, self.end = match + super().__init__(ctx, source_name, + f"{c.BUILD_INCDIR}/orderstrings/{self.start}_{self.end}.inc") + + def build(includes: List["StringInclude"]): + # Skip empty list + if len(includes) == 0: + return + + # Get context from first include (all should be equal) + ctx = includes[0].ctx + + # Build + for inc in includes: + n.build( + inc.path, + rule="orderstrings", + inputs=ctx.binary, + variables={ + "addrs" : f"{inc.start} {inc.end}" + } + ) + + def __repr__(self): + return f"StringInclude({self.start}, {self.end})" + + +class FloatInclude(GeneratedInclude): + REGEX = r'#include "(orderfloats(m?))\/([0-9a-f]{8})_([0-9a-f]{8})\.inc"' + + def __init__(self, ctx: c.SourceContext, source_name: str, match: Tuple[str]): + folder, manual, self.start, self.end = match + self.manual = manual != '' + super().__init__(ctx, source_name, + f"{c.BUILD_INCDIR}/{folder}/{self.start}_{self.end}.inc") + + def build(includes: List["FloatInclude"]): + # Skip empty list + if len(includes) == 0: + return + + # Get context from first include (all should be equal) + ctx = includes[0].ctx + + # Build + for inc in includes: + sda = "--sda " if ctx.sdata2_threshold >= 4 else "" + asm = "" if inc.manual else "--asm" + n.build( + inc.path, + rule="orderfloats", + inputs=inc.ctx.binary, + variables={ + "addrs" : f"{inc.start} {inc.end}", + "flags" : f"{sda} {asm}" + } + ) + + def __repr__(self): + return f"FloatInclude({self.start}, {self.end})" + +class DoubleInclude(GeneratedInclude): + REGEX = r'#include "orderdoubles\/([0-9a-f]{8})_([0-9a-f]{8})\.inc"' + + def __init__(self, ctx: c.SourceContext, source_name: str, match: Tuple[str]): + self.start, self.end = match + super().__init__(ctx, source_name, + f"{c.BUILD_INCDIR}/orderdoubles/{self.start}_{self.end}.inc") + + def build(includes: List["DoubleInclude"]): + # Skip empty list + if len(includes) == 0: + return + + # Get context from first include (all should be equal) + ctx = includes[0].ctx + + # Build + for inc in includes: + n.build( + inc.path, + rule="orderfloats", + inputs=ctx.binary, + variables={ + "addrs" : f"{inc.start} {inc.end}", + "flags" : f"--double" + } + ) + + def __repr__(self): + return f"DoubleInclude({self.start}, {self.end})" + +class AssetInclude(GeneratedInclude): + REGEX = r'#include "assets\/(.+)\.inc"' + + def __init__(self, ctx: c.SourceContext, source_name: str, match: Tuple[str]): + self.asset = assets[match] + self.asset_path = f"{c.ASSETS}/{self.asset.path}" + assert ctx.binary == self.asset.binary, f"Tried to include from other binary" + super().__init__(ctx, source_name, f"{c.BUILD_INCDIR}/assets/{self.asset.path}.inc") + + def build(includes: List["AssetInclude"]): + # Skip empty list + if len(includes) == 0: + return + + # Build + for inc in includes: + n.build( + inc.asset_path, + rule="assetrip", + inputs=inc.asset.binary, + variables={ + "addrs" : f"{inc.asset.start:x} {inc.asset.end:x}" + } + ) + n.build( + inc.path, + rule="assetinc", + inputs=inc.asset_path + ) + + def __repr__(self): + return f"AssetInclude({self.asset})" + +class Source(ABC): + def __init__(self, decompiled: bool, src_path: str, o_path: str, + gen_includes: List[GeneratedInclude] = []): + self.decompiled = decompiled + self.src_path = src_path + self.o_path = o_path + self.o_stem = o_path[:-2] + self.i_path = o_path[:-2] + ".i" + self.gen_includes = gen_includes + + def build(self): + raise NotImplementedError + + def make(ctx: c.SourceContext, source: c.SourceDesc): + if isinstance(source, str): + print(source) + ext = source.split('.')[-1].lower() + if ext in ("c", "cpp", "cxx", "cc"): + return CSource(ctx, source) + elif ext == "s": + return AsmSource(ctx, source) + else: + assert 0, f"Unknown source type .{ext}" + else: + return GenAsmSource(ctx, *source) + +class GenAsmSource(Source): + def __init__(self, ctx: c.SourceContext, section: str, start: int, end: int): + self.start = start + self.end = end + self.ctx = ctx + name = f"{section}_{start:x}_{end:x}.s" + src_path = f"$builddir/asm/{name}" + super().__init__(False, src_path, src_path + ".o") + + # Add ctors to forcefiles + if section == ".ctors": + forcefiles.append(name + ".o") + + + def build(self): + n.build( + self.src_path, + rule = "disasm_slice", + inputs = [self.ctx.binary, self.ctx.labels, self.ctx.relocs], + implicit = [c.GAME_SYMBOLS, self.ctx.disasm_overrides], + variables = { + "slice" : f"{self.start:x} {self.end:x}", + "disasmflags" : f"-m {c.GAME_SYMBOLS} -o {self.ctx.disasm_overrides}" + } + ) + n.build( + self.o_path, + rule="as", + inputs=self.src_path + ) + def batch_build(sources: List["GenAsmSource"], batch_size=20): + # TODO: configure batch size based on cpu core count + + # Skip empty list + if len(sources) == 0: + return + + # Get context from first include (all should be equal) + ctx = sources[0].ctx + + for src in sources: + n.build( + src.o_path, + rule="as", + inputs=src.src_path + ) + + while len(sources) > 0: + batch, sources = sources[:batch_size], sources[batch_size:] + n.build( + [src.src_path for src in batch], + rule = "disasm_slice", + inputs = [ctx.binary, ctx.labels, ctx.relocs], + implicit = [c.GAME_SYMBOLS, ctx.disasm_overrides], + variables = { + "slice" : ' '.join( + f"{src.start:x} {src.end:x}" + for src in batch + ), + "disasmflags" : f"-m {c.GAME_SYMBOLS} -o {ctx.disasm_overrides}" + } + ) + +class AsmSource(Source): + def __init__(self, ctx: c.SourceContext, path: str): + super().__init__(True, path, f"$builddir/{path}.o") + + def build(self): + n.build( + self.o_path, + rule = "as", + inputs = self.src_path + ) + +class CSource(Source): + def __init__(self, ctx: c.SourceContext, path: str): + if path.startswith("src/dolphin/"): + self.cflags = c.SDK_FLAGS + self.cc = c.OCC + self.frank = True + elif path.startswith("src/JSystem/"): + self.cflags = c.JSYSTEM_CFLAGS + self.cc = c.CC + self.frank = False + elif path.startswith("src/jaudio_NES"): + self.cc = c.CC + self.cflags = c.DOL_CPPFLAGS + self.frank = False + else: + self.cflags = ctx.cflags + self.cc = c.CC + self.frank = False + self.iconv_path = f"$builddir/iconv/{path}" + + # Find generated includes + with open(path, encoding="utf-8") as f: + gen_includes = GeneratedInclude.find(ctx, path, f.read()) + + self.s_path = f"$builddir/{path}.s" + super().__init__(True, path, f"$builddir/{path}.o", gen_includes) + + def build(self): + n.build( + self.iconv_path, + rule="iconv", + inputs=self.src_path + ) + #n.build( + # self.o_path, + # rule = "cc", + # inputs = self.iconv_path, + # implicit = [inc.path for inc in self.gen_includes], + # variables = { + # "cc" : self.cc, + # "cflags" : self.cflags + ' ' + c.PREPROCESS_CFLAGS + # } + #) + #print(self.i_path) + + if self.frank == True: + #print(f"python3 franklite.py {self.o_path} {self.o_path}") + n.build( + self.o_path, + rule = "frank", + inputs = self.iconv_path, + implicit = [inc.path for inc in self.gen_includes], + variables = { + "cc" : self.cc, + "cflags" : self.cflags, + #"basefile" : self.o_path + } + ) + else: + n.build( + self.o_path, + rule = "cc", + inputs = self.iconv_path, + implicit = [inc.path for inc in self.gen_includes], + variables = { + "cc" : self.cc, + "cflags" : self.cflags + } + ) + # Optional manual debug target + n.build( + self.s_path, + rule = "ccs", + inputs = self.iconv_path, + implicit = [inc.path for inc in self.gen_includes], + variables = { + "cflags" : self.cflags + } + ) + +def load_sources(ctx: c.SourceContext): + raw = c.get_cmd_stdout( + f"{c.SLICES} {ctx.binary} {ctx.slices} -o -p {ctx.srcdir}/" + ) + return [Source.make(ctx, s) for s in json.loads(raw)] + +def find_gen_includes(sources: List[Source]): + ret = defaultdict(list) + for source in sources: + if not isinstance(source, CSource): + continue + + for inc in source.gen_includes: + ret[type(inc)].append(inc) + + return ret + +def make_asm_list(path: str, asm_includes: List[AsmInclude]): + with open(path, 'wb') as f: + pickle.dump( + [ + int(inc.addr, 16) + for inc in asm_includes + ], + f + ) + +forcefiles = [] + +dol_sources = load_sources(c.DOL_CTX) +dol_gen_includes = find_gen_includes(dol_sources) +make_asm_list(c.DOL_ASM_LIST, dol_gen_includes[AsmInclude]) + +rel_sources = load_sources(c.REL_CTX) +rel_gen_includes = find_gen_includes(rel_sources) +make_asm_list(c.REL_ASM_LIST, rel_gen_includes[AsmInclude]) + +########## +# Builds # +########## + +n.build( + c.EXTERNS, + rule = "relextern", + inputs = c.REL_YML +) + +n.build( + [c.REL_LABELS, c.REL_RELOCS], + rule = "analyse", + inputs = c.REL_YML, + implicit = [c.ANALYSIS_OVERRIDES, c.EXTERNS], + variables = { + "analysisflags" : "$ppcdis_analysis_flags" + } +) + +n.build( + [c.DOL_LABELS, c.DOL_RELOCS], + rule = "analyse", + inputs = c.DOL_YML, + implicit = [c.ANALYSIS_OVERRIDES, c.EXTERNS], + variables = { + "analysisflags" : f"$ppcdis_analysis_flags" + } +) + +for cl, includes in dol_gen_includes.items(): + cl.build(includes) + +for cl, includes in rel_gen_includes.items(): + cl.build(includes) + +dol_gen_asm = [] +for source in dol_sources: + if isinstance(source, GenAsmSource): + dol_gen_asm.append(source) + else: + source.build() +GenAsmSource.batch_build(dol_gen_asm) + +rel_gen_asm = [] +for source in rel_sources: + if isinstance(source, GenAsmSource): + rel_gen_asm.append(source) + else: + source.build() +GenAsmSource.batch_build(rel_gen_asm) + +n.build( + c.DOL_LCF_TEMP, + rule="forceactivegen", + inputs=[c.DOL_LCF_TEMPLATE, c.DOL_YML, c.DOL_LABELS, c.GAME_SYMBOLS, c.EXTERNS] +) + +n.build( + c.DOL_LCF, + rule="forcefiles", + inputs=c.DOL_LCF_TEMP, + variables={ + "forcefiles" : ' '.join(forcefiles) + } +) + +n.build( + c.DOL_ELF, + rule="ld", + inputs=[s.o_path for s in dol_sources], + implicit=c.DOL_LCF, + implicit_outputs=c.DOL_MAP, + variables={ + "map" : c.DOL_MAP, + "lcf" : c.DOL_LCF + } +) + +n.build( + c.DOL_OUT, + rule="elf2dol", + inputs=c.DOL_ELF, +) + +n.build( + c.DOL_OK, + rule = "sha1sum", + inputs = c.DOL_SHA, + implicit = [c.DOL_OUT] +) +n.default(c.DOL_OK) + +n.build( + c.REL_PLF, + rule="ld", + inputs=[s.o_path for s in rel_sources], + implicit=c.REL_LCF, + implicit_outputs=c.REL_MAP, + variables={ + "map" : c.REL_MAP, + "lcf" : c.REL_LCF, + "ldflags" : c.LDFLAGS + " -r1 -m _prolog" + } +) + +n.build( + c.REL_OUT, + rule="elf2rel", + inputs=[c.REL_PLF, c.DOL_ELF], + variables={ + "flags" : f"-n 20 --name-size 0x38 -v 2 -r {c.REL_YML}" + } +) + +n.build( + c.REL_OK, + rule = "sha1sum", + inputs = c.REL_SHA, + implicit = [c.REL_OUT] +) +n.default(c.REL_OK) + +# Optional full binary disassembly +n.build( + c.DOL_FULL, + rule = "disasm", + inputs=[c.DOL_YML, c.DOL_LABELS, c.DOL_RELOCS], + implicit=[c.GAME_SYMBOLS, c.DOL_DISASM_OVERRIDES], + variables={ + "disasmflags" : f"-m {c.GAME_SYMBOLS} -o {c.REL_DISASM_OVERRIDES}" + } +) + +n.build( + c.REL_FULL, + rule = "disasm", + inputs=[c.REL_YML, c.REL_LABELS, c.REL_RELOCS], + implicit=[c.GAME_SYMBOLS, c.REL_DISASM_OVERRIDES], + variables={ + "disasmflags" : f"-m {c.GAME_SYMBOLS} -o {c.REL_DISASM_OVERRIDES}" + } +) + +########## +# Ouptut # +########## + +with open("build.ninja", 'w') as f: + f.write(outbuf.getvalue()) +n.close() diff --git a/tools/ppcdis b/tools/ppcdis index eca3f301..83260829 160000 --- a/tools/ppcdis +++ b/tools/ppcdis @@ -1 +1 @@ -Subproject commit eca3f3015a86bf0c78d1e415aeff2dda1851194f +Subproject commit 83260829c50f37dd9fe54889cf8971c8fe14c54e