From 47ca6fcd13804c65d06263265e45b6f1559c3230 Mon Sep 17 00:00:00 2001 From: Yanis002 <35189056+Yanis002@users.noreply.github.com> Date: Thu, 19 Jun 2025 23:51:42 +0200 Subject: [PATCH] sync configure.py with ph --- tools/configure.py | 304 +++++++++++++++++++++++++++++++++------------ 1 file changed, 223 insertions(+), 81 deletions(-) mode change 100644 => 100755 tools/configure.py diff --git a/tools/configure.py b/tools/configure.py old mode 100644 new mode 100755 index 999bb552..209db1bd --- a/tools/configure.py +++ b/tools/configure.py @@ -1,12 +1,15 @@ #!/usr/bin/env python3 +import json import os from pathlib import Path import argparse import sys +import subprocess +from typing import Any import ninja_syntax -from get_platform import get_platform +from get_platform import Platform, get_platform DEFAULT_WIBO_PATH = "./wibo" @@ -23,7 +26,7 @@ args = parser.parse_args() # Config GAME = "st" -DSD_VERSION = 'v0.6.0' +DSD_VERSION = 'v0.9.1' WIBO_VERSION = '0.6.16' OBJDIFF_VERSION = 'v2.7.1' MWCC_VERSION = "2.0/sp2p4" @@ -47,20 +50,32 @@ CC_FLAGS = " ".join([ "-nolink", # Do not link "-msgstyle gcc", # Use GCC-like messages (some IDEs will make file names clickable) ]) +# Passed to all modules and final arm9.o link LD_FLAGS = " ".join([ "-proc arm946e", # Target processor + "-dead", # Strip unused code "-nostdlib", # No C/C++ standard library "-interworking", # Enable ARM/Thumb interworking - "-m Entry", # Set entry function "-map closure,unused", # Generate map file "-msgstyle gcc", # Use GCC-like messages (some IDEs will make file names clickable) ]) +# Only passed to the module links +MODULE_LD_FLAGS = " ".join([ + "-library", # Link as a static library +]) +# Only passed to the final arm9.o link +ARM9_LD_FLAGS = " ".join([ + "-m Entry", # Set entry function +]) DSD_OBJDIFF_ARGS = " ".join([ "--scratch", # Metadata for creating decomp.me scratches f"--compiler {DECOMP_ME_COMPILER}", # decomp.me compiler name f'--c-flags "{CC_FLAGS} -lang=c++"',# decomp.me compiler flags "--custom-make ninja", # Command for rebuilding files ]) +DSD_BASE_FLAGS = " ".join([ + "--force-color", # Force color output +]) # Paths @@ -80,7 +95,7 @@ mwcc_path = mwcc_root / MWCC_VERSION # Includes includes = [ - str(root_path / "include") + root_path / "include" ] for root, dirs, _ in os.walk(libs_path): for dir in dirs: @@ -103,7 +118,7 @@ PYTHON = sys.executable class Project: - def __init__(self, game_version: str): + def __init__(self, game_version: str, *, platform: Platform, delinks_json: Any | None): self.game_version = game_version '''Version of the game''' self.game_config = config_path / game_version @@ -113,6 +128,11 @@ class Project: print(f"Version '{game_version}' not recognized") exit(1) + self.platform = platform + '''Host platform information''' + self.delinks_json = delinks_json + '''Delinks JSON data from dsd''' + self.game_build = build_path / game_version '''Path to build directory''' self.game_extract = extract_path / game_version @@ -149,27 +169,67 @@ class Project: for source_file in get_c_cpp_files([src_path, libs_path]) ] - def arm9_lcf(self) -> Path: - return self.game_build / "linker_script.lcf" - - def arm9_objects_txt(self) -> Path: - return self.game_build / "objects.txt" - - def arm9_delink_yaml(self) -> Path: - return self.game_build / "delinks" / "delink.yaml" - def arm9_o(self) -> Path: return self.game_build / "arm9.o" - def arm9_delinks(self) -> Path: - return self.game_build / "delinks" + def arm9_disassembly_dir(self) -> Path: + return self.game_build / "asm" def objdiff_report(self) -> Path: return self.game_build / "report.json" + def modules(self) -> list[Any]: + if self.delinks_json is None: + return [] + return self.delinks_json['modules'] + + def delink_files(self) -> list[str]: + if self.delinks_json is None: + return [] + return [file['delink_file'] for module in self.delinks_json['modules'] for file in module['files']] + + def arm9_lcf_file(self) -> str: + if self.delinks_json is None: + return "" + return self.delinks_json['arm9_lcf_file'] + + def module_lcf_files(self) -> list[str]: + if self.delinks_json is None: + return [] + return [module['lcf_file'] for module in self.delinks_json['modules']] + + +def can_run_dsd() -> bool: + try: + output = subprocess.run([DSD, "--version"], capture_output=True, text=True, check=True) + version = output.stdout.strip().split(" ")[-1] + if not version.startswith("v"): + version = "v" + version + + # If it's not the correct version, Ninja will download it and then rerun this script + return version == DSD_VERSION + except subprocess.CalledProcessError: + return False + except FileNotFoundError: + return False + def main(): - project = Project(args.version) + if platform is None: return + + delinks_json = None + if can_run_dsd(): + out = subprocess.run([ + DSD, + "--force-color", + "json", + "delinks", + "--config-path", config_path / args.version / "arm9" / "config.yaml" + ], capture_output=True, text=True, check=True) + delinks_json = json.loads(out.stdout) + + project = Project(args.version, platform=platform, delinks_json=delinks_json) + with build_ninja_path.open("w") as file: n = ninja_syntax.Writer(file) @@ -188,13 +248,19 @@ def main(): n.rule( name="extract", - command=f"{DSD} rom extract --rom $in --output-path $output_path $arm7_bios_flag" + command=f"{DSD} {DSD_BASE_FLAGS} rom extract --rom $in --output-path $output_path $arm7_bios_flag" ) n.newline() n.rule( name="delink", - command=f"{DSD} delink --config-path $config_path" + command=f"{DSD} {DSD_BASE_FLAGS} delink --config-path $config_path" + ) + n.newline() + + n.rule( + name="disassemble", + command=f"{DSD} {DSD_BASE_FLAGS} dis --config-path $config_path --asm-path $output_path --ual" ) n.newline() @@ -214,31 +280,31 @@ def main(): n.rule( name="lcf", - command=f"{DSD} lcf -c $config_path --lcf-file $lcf_file --objects-file $objects_file" + command=f"{DSD} {DSD_BASE_FLAGS} lcf -c $config_path" ) n.newline() n.rule( name="mwld", - command=f'{WINE} "{LD}" {LD_FLAGS} @$objects_file $lcf_file -o $out' + command=f'{WINE} "{LD}" {LD_FLAGS} $extra_ld_flags $in -o $out' ) n.newline() n.rule( name="rom_config", - command=f"{DSD} rom config --elf $in --config $config_path" + command=f"{DSD} {DSD_BASE_FLAGS} rom config --elf $in --config $config_path" ) n.newline() n.rule( name="rom_build", - command=f"{DSD} rom build --config $in --rom $out $arm7_bios_flag" + command=f"{DSD} {DSD_BASE_FLAGS} rom build --config $in --rom $out $arm7_bios_flag" ) n.newline() n.rule( name="objdiff", - command=f"{DSD} objdiff --config-path $config_path {DSD_OBJDIFF_ARGS}" + command=f"{DSD} {DSD_BASE_FLAGS} objdiff --config-path $config_path {DSD_OBJDIFF_ARGS}" ) n.newline() @@ -256,13 +322,19 @@ def main(): n.rule( name="check_modules", - command=f"{DSD} check modules --config-path $config_path --fail" + command=f"{DSD} {DSD_BASE_FLAGS} check modules --config-path $config_path --fail" ) n.newline() n.rule( name="check_symbols", - command=f"{DSD} check symbols --config-path $config_path --elf-path $elf_path --fail" + command=f"{DSD} {DSD_BASE_FLAGS} check symbols --config-path $config_path --elf-path $elf_path --fail --max-lines 20" + ) + n.newline() + + n.rule( + name="apply", + command=f"{DSD} {DSD_BASE_FLAGS} apply --config-path $config_path --elf-path $elf_path" ) n.newline() @@ -272,16 +344,29 @@ def main(): ) n.newline() - add_download_tool_builds(n) + configure_cmdline = subprocess.list2cmdline(sys.argv[1:]) + n.rule( + name="configure", + command=f"{PYTHON} tools/configure.py {configure_cmdline}", + generator=True + ) + n.newline() + + add_download_tool_builds(n, project) add_extract_build(n, project) add_delink_and_lcf_builds(n, project) + add_disassemble_builds(n, project) add_mwcc_builds(n, project, mwcc_implicit) add_mwld_and_rom_builds(n, project) add_check_builds(n, project) add_objdiff_builds(n, project) + add_configure_build(n, project) + add_apply_build(n, project) + + n.default(["objdiff", "check", "sha1"]) -def add_download_tool_builds(n: ninja_syntax.Writer): +def add_download_tool_builds(n: ninja_syntax.Writer, project: Project): if args.dsd is None: n.build( rule="download_tool", @@ -312,12 +397,12 @@ def add_download_tool_builds(n: ninja_syntax.Writer): variables={ "tool": "mwccarm", "tag": "latest", - "path": tools_path, + "path": str(tools_path), }, ) n.newline() - if platform.system != "windows" and WINE == DEFAULT_WIBO_PATH: + if project.platform.system != "windows" and WINE == DEFAULT_WIBO_PATH: n.build( rule="download_tool", outputs=WINE, @@ -345,23 +430,38 @@ def add_extract_build(n: ninja_syntax.Writer, project: Project): def add_mwld_and_rom_builds(n: ninja_syntax.Writer, project: Project): - lcf_file = str(project.arm9_lcf()) - objects_file = str(project.arm9_objects_txt()) - delink_file = str(project.arm9_delink_yaml()) - elf_file = str(project.arm9_o()) - n.build( - inputs=project.source_object_files() + [lcf_file, objects_file, delink_file], - implicit=LD, - rule="mwld", - outputs=elf_file, - variables={ - "target_dir": project.game_build, - "objects_file": objects_file, - "lcf_file": lcf_file, - } - ) + n.comment("Link each module separately") + for module in project.modules(): + lcf_file = module['lcf_file'] + objects_to_link = [file['object_to_link'] for file in module['files']] + elf_file = module['elf_file'] + n.build( + inputs=objects_to_link + [lcf_file], + implicit=LD, + rule="mwld", + outputs=elf_file, + variables={ + 'extra_ld_flags': MODULE_LD_FLAGS, + } + ) n.newline() + n.comment("Link all modules together") + module_elf_files = [module['elf_file'] for module in project.modules()] + elf_file = str(project.arm9_o()) + lcf_file = project.arm9_lcf_file() + if len(module_elf_files) > 0: + n.build( + inputs=module_elf_files + [lcf_file], + implicit=LD, + rule="mwld", + outputs=elf_file, + variables={ + 'extra_ld_flags': ARM9_LD_FLAGS, + } + ) + n.newline() + n.build( inputs=elf_file, rule="phony", @@ -376,7 +476,7 @@ def add_mwld_and_rom_builds(n: ninja_syntax.Writer, project: Project): rule="rom_config", outputs=rom_config_file, variables={ - "config_path": project.arm9_config_yaml(), + "config_path": str(project.arm9_config_yaml()), } ) n.newline() @@ -408,10 +508,10 @@ def add_mwld_and_rom_builds(n: ninja_syntax.Writer, project: Project): n.newline() -def add_mwcc_builds(n: ninja_syntax.Writer, project: Project, mwcc_implicit: list[Path]): +def add_mwcc_builds(n: ninja_syntax.Writer, project: Project, mwcc_implicit: list[str]): for source_file in get_c_cpp_files([src_path, libs_path]): src_obj_path = project.game_build / source_file - cc_flags = [] + cc_flags: list[str] = [] if is_cpp(source_file): cc_flags.append("-lang=c++") elif is_c(source_file): cc_flags.append("-lang=c") n.build( @@ -447,47 +547,60 @@ def get_c_cpp_files(dirs: list[Path]): yield root / file -def is_cpp(name: str): +def is_cpp(name: str | Path): return Path(name).suffix in [".cpp"] -def is_c(name: str): +def is_c(name: str | Path): return Path(name).suffix in [".c"] def add_delink_and_lcf_builds(n: ninja_syntax.Writer, project: Project): - n.comment("Delink ELF binaries when any delinks.txt file is modified") rom_config = str(project.baserom_config()) - delinks_path = project.arm9_delinks() - n.build( - inputs=project.dsd_configs() + [rom_config], - implicit=DSD, - rule="delink", - outputs=str(delinks_path / "delink.yaml"), - variables={ - "config_path": project.arm9_config_yaml(), - } - ) - n.newline() + delink_files = project.delink_files() + if len(delink_files) > 0: + n.comment("Delink ELF binaries when any delinks.txt file is modified") + n.build( + inputs=project.dsd_configs() + [rom_config], + implicit=DSD, + rule="delink", + outputs=delink_files, + variables={ + "config_path": str(project.arm9_config_yaml()), + } + ) + n.newline() - n.build( - inputs=str(delinks_path / "delink.yaml"), - rule="phony", - outputs="delink" - ) - n.newline() + n.build( + inputs=delink_files, + rule="phony", + outputs="delink" + ) + n.newline() - lcf_file = project.arm9_lcf() - objects_file = project.arm9_objects_txt() + lcf_files = project.module_lcf_files() + [project.arm9_lcf_file()] + if len(lcf_files) > 1: + n.build( + inputs=project.delinks_files + [str(rom_config)], + implicit=DSD, + rule="lcf", + outputs=lcf_files, + variables={ + "config_path": str(project.arm9_config_yaml()), + } + ) + n.newline() + + +def add_disassemble_builds(n: ninja_syntax.Writer, project: Project): n.build( - inputs=project.delinks_files + [str(rom_config)], + inputs=project.dsd_configs(), implicit=DSD, - rule="lcf", - outputs=[str(lcf_file), str(objects_file)], + rule="disassemble", + outputs="dis", variables={ - "config_path": project.arm9_config_yaml(), - "lcf_file": lcf_file, - "objects_file": objects_file, + "config_path": str(project.arm9_config_yaml()), + "output_path": str(project.arm9_disassembly_dir()), } ) n.newline() @@ -499,7 +612,7 @@ def add_check_builds(n: ninja_syntax.Writer, project: Project): rule="check_modules", outputs="check_modules", variables={ - "config_path": project.arm9_config_yaml(), + "config_path": str(project.arm9_config_yaml()), }, ) n.newline() @@ -509,8 +622,8 @@ def add_check_builds(n: ninja_syntax.Writer, project: Project): rule="check_symbols", outputs="check_symbols", variables={ - "config_path": project.arm9_config_yaml(), - "elf_path": project.arm9_o(), + "config_path": str(project.arm9_config_yaml()), + "elf_path": str(project.arm9_o()), }, ) n.newline() @@ -530,7 +643,7 @@ def add_objdiff_builds(n: ninja_syntax.Writer, project: Project): rule="objdiff", outputs="objdiff.json", variables={ - "config_path": project.arm9_config_yaml(), + "config_path": str(project.arm9_config_yaml()), } ) n.newline() @@ -542,9 +655,10 @@ def add_objdiff_builds(n: ninja_syntax.Writer, project: Project): ) n.newline() + delink_files = [file['delink_file'] for module in project.modules() for file in module['files']] n.build( inputs=["objdiff.json"], - implicit=[OBJDIFF] + project.source_object_files(), + implicit=[OBJDIFF] + delink_files + project.source_object_files(), rule="objdiff_report", outputs=str(project.objdiff_report()), ) @@ -558,6 +672,34 @@ def add_objdiff_builds(n: ninja_syntax.Writer, project: Project): n.newline() +def add_configure_build(n: ninja_syntax.Writer, project: Project): + this_file = str(Path(__file__).resolve()) + n.build( + outputs="build.ninja", + rule="configure", + implicit=[ + this_file, + # Require dsd to exist when rerunning configure.py + DSD, + *project.dsd_configs(), + ] + ) + + +def add_apply_build(n: ninja_syntax.Writer, project: Project): + n.build( + inputs=project.dsd_configs() + [str(project.arm9_o())], + implicit=DSD, + rule="apply", + outputs="apply", + variables={ + "config_path": str(project.arm9_config_yaml()), + "elf_path": str(project.arm9_o()), + } + ) + n.newline() + + def get_config_files(game_config: Path, name: str) -> list[str]: return [ f"{root}/{file}"