diff --git a/.gitignore b/.gitignore index 6ee4c378..3411e74c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ objdiff.json build.ninja .ninja_log* .ninja_lock +.ninja_deps /wibo diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7e671659..0eb4c337 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,14 +17,13 @@ - `extract/`: Game assets, extracted from your own supplied ROM - `eur|usa/`: [`ds-rom`](https://github.com/AetiasHax/ds-rom) extract directories - `include/`: Include files -- `src/`: Source C/C++ files +- `libs/`: Source C/C++ files for libraries used by the game +- `src/`: Source C/C++ files for the game - `tools/`: Tools for this project - `mwccarm/`: Compiler toolchain - `configure.py`: Generates `build.ninja` - - `m2ctx.py`: Generates context for [decomp.me](https://decomp.me/) - `mangle.py`: Shows mangled symbol names in a given C/C++ file - `requirements.txt`: Python libraries - - `setup.py`: Sets up the project - `*.sha1`: SHA-1 digests of different versions of the game ## Decompiling diff --git a/INSTALL.md b/INSTALL.md index 525f22f6..713e9451 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -32,9 +32,6 @@ python tools/configure.py Now you can run `ninja` to build a ROM for the chosen version. -> [!IMPORTANT] -> Rerun `configure.py` often to ensure that all C/C++ code gets compiled. - > [!NOTE] > For Linux users: Wibo is used by default. If you want to use Wine instead, run `configure.py` with `-w `. @@ -49,5 +46,3 @@ ARM7 BIOS in the root directory of this repository, and verify that your dumped | --------------- | ------------------------------------------ | | `arm7_bios.bin` | `6ee830c7f552c5bf194c20a2c13d5bb44bdb5c03` | | `arm7_bios.bin` | `24f67bdea115a2c847c8813a262502ee1607b7df` | - -Now, rerun `configure.py` so it can update `build.ninja` to build a matching ROM. diff --git a/tools/configure.py b/tools/configure.py index c9cda3d7..cbbb566c 100755 --- a/tools/configure.py +++ b/tools/configure.py @@ -1,12 +1,15 @@ #!/usr/bin/env python3 +import json import os from pathlib import Path import argparse import sys +import subprocess +from typing import Any import ninja_syntax -from get_platform import get_platform +from get_platform import Platform, get_platform DEFAULT_WIBO_PATH = "./wibo" @@ -23,7 +26,7 @@ args = parser.parse_args() # Config GAME = "ph" -DSD_VERSION = 'v0.8.0' +DSD_VERSION = 'v0.9.0' WIBO_VERSION = '0.6.16' OBJDIFF_VERSION = 'v2.7.1' MWCC_VERSION = "2.0/sp1p5" @@ -47,14 +50,23 @@ CC_FLAGS = " ".join([ "-nolink", # Do not link "-msgstyle gcc", # Use GCC-like messages (some IDEs will make file names clickable) ]) +# Passed to all modules and final arm9.o link LD_FLAGS = " ".join([ "-proc arm946e", # Target processor + "-dead", # Strip unused code "-nostdlib", # No C/C++ standard library "-interworking", # Enable ARM/Thumb interworking - "-m Entry", # Set entry function "-map closure,unused", # Generate map file "-msgstyle gcc", # Use GCC-like messages (some IDEs will make file names clickable) ]) +# Only passed to the module links +MODULE_LD_FLAGS = " ".join([ + "-library", # Link as a static library +]) +# Only passed to the final arm9.o link +ARM9_LD_FLAGS = " ".join([ + "-m Entry", # Set entry function +]) DSD_OBJDIFF_ARGS = " ".join([ "--scratch", # Metadata for creating decomp.me scratches f"--compiler {DECOMP_ME_COMPILER}", # decomp.me compiler name @@ -83,7 +95,7 @@ mwcc_path = mwcc_root / MWCC_VERSION # Includes includes = [ - str(root_path / "include") + root_path / "include" ] for root, dirs, _ in os.walk(libs_path): for dir in dirs: @@ -106,7 +118,7 @@ PYTHON = sys.executable class Project: - def __init__(self, game_version: str): + def __init__(self, game_version: str, *, platform: Platform, delinks_json: Any | None): self.game_version = game_version '''Version of the game''' self.game_config = config_path / game_version @@ -116,6 +128,11 @@ class Project: print(f"Version '{game_version}' not recognized") exit(1) + self.platform = platform + '''Host platform information''' + self.delinks_json = delinks_json + '''Delinks JSON data from dsd''' + self.game_build = build_path / game_version '''Path to build directory''' self.game_extract = extract_path / game_version @@ -152,27 +169,67 @@ class Project: for source_file in get_c_cpp_files([src_path, libs_path]) ] - def arm9_lcf(self) -> Path: - return self.game_build / "linker_script.lcf" - - def arm9_objects_txt(self) -> Path: - return self.game_build / "objects.txt" - - def arm9_delink_yaml(self) -> Path: - return self.game_build / "delinks" / "delink.yaml" - def arm9_o(self) -> Path: return self.game_build / "arm9.o" - def arm9_delinks(self) -> Path: - return self.game_build / "delinks" + def arm9_disassembly_dir(self) -> Path: + return self.game_build / "asm" def objdiff_report(self) -> Path: return self.game_build / "report.json" + def modules(self) -> list[Any]: + if self.delinks_json is None: + return [] + return self.delinks_json['modules'] + + def delink_files(self) -> list[str]: + if self.delinks_json is None: + return [] + return [file['delink_file'] for module in self.delinks_json['modules'] for file in module['files']] + + def arm9_lcf_file(self) -> str: + if self.delinks_json is None: + return "" + return self.delinks_json['arm9_lcf_file'] + + def module_lcf_files(self) -> list[str]: + if self.delinks_json is None: + return [] + return [module['lcf_file'] for module in self.delinks_json['modules']] + + +def can_run_dsd() -> bool: + try: + output = subprocess.run([DSD, "--version"], capture_output=True, text=True, check=True) + version = output.stdout.strip().split(" ")[-1] + if not version.startswith("v"): + version = "v" + version + + # If it's not the correct version, Ninja will download it and then rerun this script + return version == DSD_VERSION + except subprocess.CalledProcessError: + return False + except FileNotFoundError: + return False + def main(): - project = Project(args.version) + if platform is None: return + + delinks_json = None + if can_run_dsd(): + out = subprocess.run([ + DSD, + "--force-color", + "json", + "delinks", + "--config-path", config_path / args.version / "arm9" / "config.yaml" + ], capture_output=True, text=True, check=True) + delinks_json = json.loads(out.stdout) + + project = Project(args.version, platform=platform, delinks_json=delinks_json) + with build_ninja_path.open("w") as file: n = ninja_syntax.Writer(file) @@ -201,6 +258,12 @@ def main(): ) n.newline() + n.rule( + name="disassemble", + command=f"{DSD} {DSD_BASE_FLAGS} dis --config-path $config_path --asm-path $output_path --ual" + ) + n.newline() + # -MMD excludes all includes instead of just system includes for some reason, so use -MD instead. mwcc_cmd = f'{WINE} "{CC}" {CC_FLAGS} {CC_INCLUDES} $cc_flags -d $game_version -MD -c $in -o $basedir' mwcc_implicit = [CC] @@ -217,13 +280,13 @@ def main(): n.rule( name="lcf", - command=f"{DSD} {DSD_BASE_FLAGS} lcf -c $config_path --lcf-file $lcf_file --objects-file $objects_file" + command=f"{DSD} {DSD_BASE_FLAGS} lcf -c $config_path" ) n.newline() n.rule( name="mwld", - command=f'{WINE} "{LD}" {LD_FLAGS} @$objects_file $lcf_file -o $out' + command=f'{WINE} "{LD}" {LD_FLAGS} $extra_ld_flags $in -o $out' ) n.newline() @@ -265,7 +328,13 @@ def main(): n.rule( name="check_symbols", - command=f"{DSD} {DSD_BASE_FLAGS} check symbols --config-path $config_path --elf-path $elf_path --fail" + command=f"{DSD} {DSD_BASE_FLAGS} check symbols --config-path $config_path --elf-path $elf_path --fail --max-lines 20" + ) + n.newline() + + n.rule( + name="apply", + command=f"{DSD} {DSD_BASE_FLAGS} apply --config-path $config_path --elf-path $elf_path" ) n.newline() @@ -275,16 +344,29 @@ def main(): ) n.newline() - add_download_tool_builds(n) + configure_cmdline = subprocess.list2cmdline(sys.argv[1:]) + n.rule( + name="configure", + command=f"{PYTHON} tools/configure.py {configure_cmdline}", + generator=True + ) + n.newline() + + add_download_tool_builds(n, project) add_extract_build(n, project) add_delink_and_lcf_builds(n, project) + add_disassemble_builds(n, project) add_mwcc_builds(n, project, mwcc_implicit) add_mwld_and_rom_builds(n, project) add_check_builds(n, project) add_objdiff_builds(n, project) + add_configure_build(n, project) + add_apply_build(n, project) + + n.default(["objdiff", "check", "sha1"]) -def add_download_tool_builds(n: ninja_syntax.Writer): +def add_download_tool_builds(n: ninja_syntax.Writer, project: Project): if args.dsd is None: n.build( rule="download_tool", @@ -315,12 +397,12 @@ def add_download_tool_builds(n: ninja_syntax.Writer): variables={ "tool": "mwccarm", "tag": "latest", - "path": tools_path, + "path": str(tools_path), }, ) n.newline() - if platform.system != "windows" and WINE == DEFAULT_WIBO_PATH: + if project.platform.system != "windows" and WINE == DEFAULT_WIBO_PATH: n.build( rule="download_tool", outputs=WINE, @@ -348,23 +430,38 @@ def add_extract_build(n: ninja_syntax.Writer, project: Project): def add_mwld_and_rom_builds(n: ninja_syntax.Writer, project: Project): - lcf_file = str(project.arm9_lcf()) - objects_file = str(project.arm9_objects_txt()) - delink_file = str(project.arm9_delink_yaml()) - elf_file = str(project.arm9_o()) - n.build( - inputs=project.source_object_files() + [lcf_file, objects_file, delink_file], - implicit=LD, - rule="mwld", - outputs=elf_file, - variables={ - "target_dir": project.game_build, - "objects_file": objects_file, - "lcf_file": lcf_file, - } - ) + n.comment("Link each module separately") + for module in project.modules(): + lcf_file = module['lcf_file'] + objects_to_link = [file['object_to_link'] for file in module['files']] + elf_file = module['elf_file'] + n.build( + inputs=objects_to_link + [lcf_file], + implicit=LD, + rule="mwld", + outputs=elf_file, + variables={ + 'extra_ld_flags': MODULE_LD_FLAGS, + } + ) n.newline() + n.comment("Link all modules together") + module_elf_files = [module['elf_file'] for module in project.modules()] + elf_file = str(project.arm9_o()) + lcf_file = project.arm9_lcf_file() + if len(module_elf_files) > 0: + n.build( + inputs=module_elf_files + [lcf_file], + implicit=LD, + rule="mwld", + outputs=elf_file, + variables={ + 'extra_ld_flags': ARM9_LD_FLAGS, + } + ) + n.newline() + n.build( inputs=elf_file, rule="phony", @@ -379,7 +476,7 @@ def add_mwld_and_rom_builds(n: ninja_syntax.Writer, project: Project): rule="rom_config", outputs=rom_config_file, variables={ - "config_path": project.arm9_config_yaml(), + "config_path": str(project.arm9_config_yaml()), } ) n.newline() @@ -411,10 +508,10 @@ def add_mwld_and_rom_builds(n: ninja_syntax.Writer, project: Project): n.newline() -def add_mwcc_builds(n: ninja_syntax.Writer, project: Project, mwcc_implicit: list[Path]): +def add_mwcc_builds(n: ninja_syntax.Writer, project: Project, mwcc_implicit: list[str]): for source_file in get_c_cpp_files([src_path, libs_path]): src_obj_path = project.game_build / source_file - cc_flags = [] + cc_flags: list[str] = [] if is_cpp(source_file): cc_flags.append("-lang=c++") elif is_c(source_file): cc_flags.append("-lang=c") n.build( @@ -450,47 +547,60 @@ def get_c_cpp_files(dirs: list[Path]): yield root / file -def is_cpp(name: str): +def is_cpp(name: str | Path): return Path(name).suffix in [".cpp"] -def is_c(name: str): +def is_c(name: str | Path): return Path(name).suffix in [".c"] def add_delink_and_lcf_builds(n: ninja_syntax.Writer, project: Project): - n.comment("Delink ELF binaries when any delinks.txt file is modified") rom_config = str(project.baserom_config()) - delinks_path = project.arm9_delinks() - n.build( - inputs=project.dsd_configs() + [rom_config], - implicit=DSD, - rule="delink", - outputs=str(delinks_path / "delink.yaml"), - variables={ - "config_path": project.arm9_config_yaml(), - } - ) - n.newline() + delink_files = project.delink_files() + if len(delink_files) > 0: + n.comment("Delink ELF binaries when any delinks.txt file is modified") + n.build( + inputs=project.dsd_configs() + [rom_config], + implicit=DSD, + rule="delink", + outputs=delink_files, + variables={ + "config_path": str(project.arm9_config_yaml()), + } + ) + n.newline() - n.build( - inputs=str(delinks_path / "delink.yaml"), - rule="phony", - outputs="delink" - ) - n.newline() + n.build( + inputs=delink_files, + rule="phony", + outputs="delink" + ) + n.newline() - lcf_file = project.arm9_lcf() - objects_file = project.arm9_objects_txt() + lcf_files = project.module_lcf_files() + [project.arm9_lcf_file()] + if len(lcf_files) > 1: + n.build( + inputs=project.delinks_files + [str(rom_config)], + implicit=DSD, + rule="lcf", + outputs=lcf_files, + variables={ + "config_path": str(project.arm9_config_yaml()), + } + ) + n.newline() + + +def add_disassemble_builds(n: ninja_syntax.Writer, project: Project): n.build( - inputs=project.delinks_files + [str(rom_config)], + inputs=project.dsd_configs(), implicit=DSD, - rule="lcf", - outputs=[str(lcf_file), str(objects_file)], + rule="disassemble", + outputs="dis", variables={ - "config_path": project.arm9_config_yaml(), - "lcf_file": lcf_file, - "objects_file": objects_file, + "config_path": str(project.arm9_config_yaml()), + "output_path": str(project.arm9_disassembly_dir()), } ) n.newline() @@ -502,7 +612,7 @@ def add_check_builds(n: ninja_syntax.Writer, project: Project): rule="check_modules", outputs="check_modules", variables={ - "config_path": project.arm9_config_yaml(), + "config_path": str(project.arm9_config_yaml()), }, ) n.newline() @@ -512,8 +622,8 @@ def add_check_builds(n: ninja_syntax.Writer, project: Project): rule="check_symbols", outputs="check_symbols", variables={ - "config_path": project.arm9_config_yaml(), - "elf_path": project.arm9_o(), + "config_path": str(project.arm9_config_yaml()), + "elf_path": str(project.arm9_o()), }, ) n.newline() @@ -533,7 +643,7 @@ def add_objdiff_builds(n: ninja_syntax.Writer, project: Project): rule="objdiff", outputs="objdiff.json", variables={ - "config_path": project.arm9_config_yaml(), + "config_path": str(project.arm9_config_yaml()), } ) n.newline() @@ -545,9 +655,10 @@ def add_objdiff_builds(n: ninja_syntax.Writer, project: Project): ) n.newline() + delink_files = [file['delink_file'] for module in project.modules() for file in module['files']] n.build( inputs=["objdiff.json"], - implicit=[OBJDIFF] + project.source_object_files(), + implicit=[OBJDIFF] + delink_files + project.source_object_files(), rule="objdiff_report", outputs=str(project.objdiff_report()), ) @@ -561,6 +672,34 @@ def add_objdiff_builds(n: ninja_syntax.Writer, project: Project): n.newline() +def add_configure_build(n: ninja_syntax.Writer, project: Project): + this_file = str(Path(__file__).resolve()) + n.build( + outputs="build.ninja", + rule="configure", + implicit=[ + this_file, + # Require dsd to exist when rerunning configure.py + DSD, + *project.dsd_configs(), + ] + ) + + +def add_apply_build(n: ninja_syntax.Writer, project: Project): + n.build( + inputs=project.dsd_configs() + [str(project.arm9_o())], + implicit=DSD, + rule="apply", + outputs="apply", + variables={ + "config_path": str(project.arm9_config_yaml()), + "elf_path": str(project.arm9_o()), + } + ) + n.newline() + + def get_config_files(game_config: Path, name: str) -> list[str]: return [ f"{root}/{file}"