From b80acf3455f42bf45ec68b378fe71e81c78a7326 Mon Sep 17 00:00:00 2001 From: Yanis <35189056+Yanis002@users.noreply.github.com> Date: Sat, 17 Jan 2026 19:21:42 +0100 Subject: [PATCH] Update tools (#21) * improve configure.py part 1 * compile settings per-file instead of global * remove dont_reuse_strings in sources * multiversion support * update docs * report filename change * fixed objdiff not being able to create context file * add version define to decompme cflags/m2ctx --- .github/workflows/build.yml | 4 +- INSTALL.md | 6 +- libs/c/src/ansi_fp.c | 2 - libs/c/src/code_02038bac.c | 0 libs/c/src/wprintf.c | 2 - src/018_StartUp/GameModeStartUp.cpp | 2 - src/018_StartUp/StartUpInitializers.cpp | 2 - src/019_MainSelect/FileSelectMain.cpp | 2 - src/019_MainSelect/FileSelectSubScreen.cpp | 2 - src/025_Title/TitleScreen.cpp | 2 - src/Main/System/SysNew.cpp | 2 - tools/configure.py | 893 +++++--------------- tools/m2ctx.py | 2 + tools/objdiff_config.py | 117 +++ tools/project.py | 920 +++++++++++++++++++++ 15 files changed, 1251 insertions(+), 707 deletions(-) delete mode 100644 libs/c/src/code_02038bac.c create mode 100644 tools/objdiff_config.py create mode 100644 tools/project.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b7c4c830..5500f6a4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,8 +26,8 @@ jobs: - name: Build run: | - python tools/configure.py ${{ matrix.version }} -w wibo --compiler /mwccarm --no-extract - ninja arm9 report check + python tools/configure.py -v ${{ matrix.version }} -w wibo --compiler /mwccarm --no-extract + ninja arm9_${{ matrix.version }} report_${{ matrix.version }} check_${{ matrix.version }} - name: Upload report uses: actions/upload-artifact@v4 diff --git a/INSTALL.md b/INSTALL.md index 02871b54..e9df2ed3 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -26,7 +26,11 @@ pre-commit install ``` 5. Run the Ninja configure script: ```shell -python tools/configure.py +python tools/configure.py +``` +By default this will configure for any version that has a baserom in the `extract` folder (see below), to configure for one specific version: +```shell +python tools/configure.py [--version | -v] ``` 6. Put one or more base ROMs in the [`/extract/`](/extract/README.md) directory of this repository. diff --git a/libs/c/src/ansi_fp.c b/libs/c/src/ansi_fp.c index e4790a19..dfe2b8a2 100644 --- a/libs/c/src/ansi_fp.c +++ b/libs/c/src/ansi_fp.c @@ -3,8 +3,6 @@ #include #include -#pragma dont_reuse_strings off - typedef unsigned long long d_int; #define SIGDIGLEN 32 diff --git a/libs/c/src/code_02038bac.c b/libs/c/src/code_02038bac.c deleted file mode 100644 index e69de29b..00000000 diff --git a/libs/c/src/wprintf.c b/libs/c/src/wprintf.c index 729a5d47..dcf4e83e 100644 --- a/libs/c/src/wprintf.c +++ b/libs/c/src/wprintf.c @@ -10,8 +10,6 @@ #include -#pragma dont_reuse_strings off - #define MAX_SIG_DIG 32 #define LDBL_MANT_DIG 24 diff --git a/src/018_StartUp/GameModeStartUp.cpp b/src/018_StartUp/GameModeStartUp.cpp index 6319142a..da9bd94c 100644 --- a/src/018_StartUp/GameModeStartUp.cpp +++ b/src/018_StartUp/GameModeStartUp.cpp @@ -11,8 +11,6 @@ #include "Unknown/UnkStruct_ov000_020b50c0.hpp" #include "regs.h" -#pragma dont_reuse_strings off - extern "C" { void func_02013768(void *param1); void func_020141dc(unk16 *param1); diff --git a/src/018_StartUp/StartUpInitializers.cpp b/src/018_StartUp/StartUpInitializers.cpp index 443d0b79..e7bf278f 100644 --- a/src/018_StartUp/StartUpInitializers.cpp +++ b/src/018_StartUp/StartUpInitializers.cpp @@ -20,8 +20,6 @@ #include "regs.h" #include "versions.h" -#pragma dont_reuse_strings off - extern "C" { unk32 func_02014fe0(); unk16 func_02026738(); diff --git a/src/019_MainSelect/FileSelectMain.cpp b/src/019_MainSelect/FileSelectMain.cpp index 27f19bc1..23bd9531 100644 --- a/src/019_MainSelect/FileSelectMain.cpp +++ b/src/019_MainSelect/FileSelectMain.cpp @@ -9,8 +9,6 @@ #include "Unknown/UnkStruct_ov000_020b5214.hpp" #include "regs.h" -#pragma dont_reuse_strings off - extern "C" { void func_020249d4(void *pReg, unk32 param1, unk32 param2, unk32 param3, unk32 param4); void func_ov000_02062e44(void *param1, void *param2); diff --git a/src/019_MainSelect/FileSelectSubScreen.cpp b/src/019_MainSelect/FileSelectSubScreen.cpp index 2bdc1e0e..44a6cbc4 100644 --- a/src/019_MainSelect/FileSelectSubScreen.cpp +++ b/src/019_MainSelect/FileSelectSubScreen.cpp @@ -11,8 +11,6 @@ void func_020249d4(void *pReg, unk32 param1, unk32 param2, unk32 param3, unk32 p void func_020275e8(); }; -#pragma dont_reuse_strings off - FileSelectOptionsManager *gpFSOptionsManager = NULL; static unk8 data_ov019_020fb8cc[0xDC00]; diff --git a/src/025_Title/TitleScreen.cpp b/src/025_Title/TitleScreen.cpp index c0bfa80e..916e5f68 100644 --- a/src/025_Title/TitleScreen.cpp +++ b/src/025_Title/TitleScreen.cpp @@ -11,8 +11,6 @@ #include -#pragma dont_reuse_strings off - extern "C" { void func_02018424(); void func_ov000_0205be34(void *param1, unk32 param2); diff --git a/src/Main/System/SysNew.cpp b/src/Main/System/SysNew.cpp index a2fe20d7..5cf74ac3 100644 --- a/src/Main/System/SysNew.cpp +++ b/src/Main/System/SysNew.cpp @@ -2,8 +2,6 @@ #include "System/SysFault.hpp" #include "global.h" -#pragma dont_reuse_strings off - extern "C" { void *func_02001654(void); void *func_020145b0(UnkId *, s32); diff --git a/tools/configure.py b/tools/configure.py index 4a95ae6e..f7201860 100755 --- a/tools/configure.py +++ b/tools/configure.py @@ -1,37 +1,48 @@ #!/usr/bin/env python3 import json -import os -from pathlib import Path import argparse -import sys import subprocess -from typing import Any +import glob -import ninja_syntax -from get_platform import Platform, get_platform - - -DEFAULT_WIBO_PATH = "./wibo" +from typing import List, Dict, Any +from pathlib import Path +from project import ProjectConfig, Object, process_project parser = argparse.ArgumentParser(description="Generates build.ninja") -parser.add_argument('-w', type=str, default=DEFAULT_WIBO_PATH, dest="wine", required=False, help="Path to Wine/Wibo (linux only)") +parser.add_argument('-w', type=str, default="./wibo", dest="wine", required=False, help="Path to Wine/Wibo (linux only)") parser.add_argument("--compiler", type=Path, required=False, help="Path to pre-installed compiler root directory") parser.add_argument("--no-extract", action="store_true", help="Skip extract step") parser.add_argument("--dsd", type=Path, required=False, help="Path to pre-installed dsd CLI") -parser.add_argument('version', help='Game version') +parser.add_argument("--version", "-v", help='Game version', required=False) args = parser.parse_args() +config = ProjectConfig("st", args.compiler, "dsi/1.2p1", args.wine, args.dsd, Path(__file__).resolve()) +config.dsd_tag = "v0.10.2" +config.wibo_tag = "0.6.16" +config.objdiff_tag = "v3.0.0-beta.6" -# Config -GAME = "st" -DSD_VERSION = 'v0.10.2' -WIBO_VERSION = '0.6.16' -OBJDIFF_VERSION = 'v3.0.0-beta.6' -MWCC_VERSION = "dsi/1.2p1" -DECOMP_ME_COMPILER = "mwcc_40_1027" -CC_FLAGS = " ".join([ +GAME_VERSIONS = [ + "eur", + "jp", +] + +# Only configure versions for which a baserom file exists +def version_exists(version: str) -> bool: + return glob.glob(str(Path("extract") / f"baserom_st_{version}.nds")) != [] + +if args.version is not None: + config.game_versions = [args.version] +else: + config.game_versions = [ + version + for version in GAME_VERSIONS + if version_exists(version) + ] + + +config.cflags_base = [ "-O4,p", # Optimize maximally for performance "-enum int", # Use int-sized enums "-char signed", # Char type is signed @@ -50,692 +61,198 @@ CC_FLAGS = " ".join([ "-nolink", # Do not link "-msgstyle gcc", # Use GCC-like messages (some IDEs will make file names clickable) "-ipa file", # InterProcedural Analysis -]) -# Passed to all modules and final arm9.o link -LD_FLAGS = " ".join([ +] + +config.ldflags = [ "-proc arm946e", # Target processor "-dead", # Strip unused code "-nostdlib", # No C/C++ standard library "-interworking", # Enable ARM/Thumb interworking "-map closure,unused", # Generate map file "-msgstyle gcc", # Use GCC-like messages (some IDEs will make file names clickable) -]) -# Only passed to the module links -MODULE_LD_FLAGS = " ".join([ - "-library", # Link as a static library -]) -# Only passed to the final arm9.o link -ARM9_LD_FLAGS = " ".join([ - "-m Entry", # Set entry function -]) -DSD_OBJDIFF_ARGS = " ".join([ - "--scratch", # Metadata for creating decomp.me scratches - f"--compiler {DECOMP_ME_COMPILER}", # decomp.me compiler name - f'--c-flags "{CC_FLAGS} -lang=c++"',# decomp.me compiler flags - "--custom-make ninja", # Command for rebuilding files -]) -DSD_BASE_FLAGS = " ".join([ - "--force-color", # Force color output -]) - - -# Paths -current_path = Path(__name__) -root_path = current_path.parent -build_ninja_path = root_path / "build.ninja" -arm7_bios_path = root_path / "arm7_bios.bin" -config_path = root_path / "config" -build_path = root_path / "build" -src_path = root_path / "src" -libs_path = root_path / "libs" -extract_path = root_path / "extract" -tools_path = root_path / "tools" -mwcc_root = args.compiler or tools_path / "mwccarm" -mwcc_path = mwcc_root / MWCC_VERSION - - -# Includes -includes = [ - root_path / "include" ] -for root, dirs, _ in os.walk(libs_path): - for dir in dirs: - if dir == "include": - includes.append(Path(root) / dir) -CC_INCLUDES = " ".join(f"-i {include}" for include in includes) -# Platform info -platform = get_platform() -if platform is None: - exit(1) -EXE = platform.exe -WINE = args.wine if platform.system != "windows" else "" -DSD = str(args.dsd or os.path.join('.', str(root_path / f"dsd{EXE}"))) -OBJDIFF = os.path.join('.', str(root_path / f"objdiff-cli{EXE}")) -CC = os.path.join('.', str(mwcc_path / "mwccarm.exe")) -LD = os.path.join('.', str(mwcc_path / "mwldarm.exe")) -PYTHON = sys.executable +# Helper function for Nitro libraries +def NitroLib(lib_name: str, objects: List[Object]) -> Dict[str, Any]: + return { + "lib": lib_name, + "mw_version": "dsi/1.2p1", + "src_dir": "libs/nitro/src", + "cflags": config.cflags_base, + "objects": objects, + } -class Project: - def __init__(self, game_version: str, *, platform: Platform, delinks_json: Any | None): - self.game_version = game_version - '''Version of the game''' - self.game_config = config_path / game_version - '''Root directory for dsd configs''' - - if not self.game_config.is_dir(): - print(f"Version '{game_version}' not recognized") - exit(1) - - self.platform = platform - '''Host platform information''' - self.delinks_json = delinks_json - '''Delinks JSON data from dsd''' - - self.game_build = build_path / game_version - '''Path to build directory''' - self.game_extract = extract_path / game_version - '''Path to extract directory''' - - self.delinks_files = get_config_files(self.game_config, "delinks.txt") - '''Paths to every delinks.txt file''' - self.relocs_files = get_config_files(self.game_config, "relocs.txt") - '''Paths to every relocs.txt file''' - self.symbols_files = get_config_files(self.game_config, "symbols.txt") - '''Paths to every symbols.txt file''' - - def dsd_configs(self) -> list[str]: - return self.delinks_files + self.relocs_files + self.symbols_files - - def arm9_config_yaml(self) -> Path: - return self.game_config / "arm9" / "config.yaml" - - def baserom(self) -> Path: - return extract_path / f'baserom_{GAME}_{self.game_version}.nds' - - def build_rom(self) -> str: - return f"{GAME}_{self.game_version}.nds" - - def baserom_config(self) -> Path: - return self.game_extract / 'config.yaml' - - def build_rom_config(self) -> Path: - return self.game_build / "build" / "rom_config.yaml" - - def source_object_files(self) -> list[str]: - files: list[str] = [] - for source_file in get_c_cpp_files([src_path, libs_path]): - src_obj_path = self.game_build / source_file - files.append(str(src_obj_path.with_suffix(".o"))) - return files - - def arm9_o(self) -> Path: - return self.game_build / "arm9.o" - - def arm9_disassembly_dir(self) -> Path: - return self.game_build / "asm" - - def objdiff_report(self) -> Path: - return self.game_build / "report.json" - - def files(self) -> list[dict[str, str]]: - if self.delinks_json is None: - return [] - return self.delinks_json['files'] - - def delink_files(self) -> list[str]: - delink_files = [file['delink_file'] for file in self.files()] - return list(set(delink_files)) - - def arm9_lcf_file(self) -> str: - if self.delinks_json is None: - return "" - return self.delinks_json['arm9_lcf_file'] - - def arm9_objects_file(self) -> str: - if self.delinks_json is None: - return "" - return self.delinks_json['arm9_objects_file'] +# Helper function for libc libraries +def LibC(lib_name: str, objects: List[Object]) -> Dict[str, Any]: + return { + "lib": lib_name, + "mw_version": "dsi/1.2p1", + "src_dir": "libs/c/src", + "cflags": [*config.cflags_base, "-str reuse"], + "objects": objects, + } -def check_can_run_dsd() -> bool: - try: - output = subprocess.run([DSD, "--version"], capture_output=True, text=True, check=True) - version = output.stdout.strip().split(" ")[-1] - if not version.startswith("v"): - version = "v" + version +# Helper function for libcpp libraries +def LibCPP(lib_name: str, objects: List[Object]) -> Dict[str, Any]: + return { + "lib": lib_name, + "mw_version": "dsi/1.2p1", + "src_dir": "libs/cpp/src", + "cflags": config.cflags_base, + "objects": objects, + } - # If it's not the correct version, Ninja will download it and then rerun this script - return version == DSD_VERSION - except subprocess.CalledProcessError: - return False - except FileNotFoundError: - return False + +# Helper function for overlays and similar modules +def GameLib(lib_name: str, objects: List[Object]) -> Dict[str, Any]: + return { + "lib": lib_name, + "mw_version": "dsi/1.2p1", + "cflags": [*config.cflags_base, "-str reuse"], + "objects": objects, + } + + +config.auto_add_sources = False +config.warn_missing_source = True + +config.libs = [ + GameLib( + "Main", + [ + Object("Main/Main.cpp"), + Object("Main/System/SysNew.cpp"), + Object("Main/System/OverlayManager.cpp"), + Object("Main/func_02017ea4.cpp"), + ] + ), + GameLib( + "Overlay 0", + [ + Object("000_Second/Actor/Actor.cpp"), + Object("000_Second/Actor/ActorManager.cpp"), + Object("000_Second/Actor/ActorUnk_ov000_020a8bb0.cpp"), + Object("000_Second/Item/ItemManager.cpp"), + Object("000_Second/Item/TreasureManager.cpp"), + ] + ), + GameLib( + "Overlay 1", + [ + Object("001_SceneInit/Actor/ActorManager_001.cpp"), + ] + ), + GameLib( + "Overlay 18", + [ + Object("018_StartUp/GameModeStartUp.cpp"), + Object("018_StartUp/StartUpInitializers.cpp"), + ] + ), + GameLib( + "Overlay 19", + [ + Object("019_MainSelect/GameModeFileSelect.cpp"), + Object("019_MainSelect/FileSelectManager.cpp"), + Object("019_MainSelect/019_UnkSystem1_ov019_Derived1.cpp"), + Object("019_MainSelect/FileSelectMain.cpp"), + Object("019_MainSelect/019_UnkSubStruct9.cpp"), + Object("019_MainSelect/FileSelectOptions.cpp"), + Object("019_MainSelect/FileSelectMicTest.cpp"), + Object("019_MainSelect/FileSelectSubScreen.cpp"), + Object("019_MainSelect/FileSelectManager_160.cpp"), + Object("019_MainSelect/FileSelectManager_164.cpp"), + Object("019_MainSelect/019_UnkSystem1_ov019_Derived2.cpp"), + Object("019_MainSelect/019_UnkSystem1_ov019_Derived3.cpp"), + Object("019_MainSelect/019_SaveManager.cpp"), + ] + ), + GameLib( + "Overlay 25", + [ + Object("025_Title/GameModeTitleScreen.cpp"), + Object("025_Title/TitleScreenManager.cpp"), + Object("025_Title/TitleScreen.cpp"), + ] + ), + GameLib( + "Overlay 31", + [ + Object("031_Land/Actor/ActorRupee.cpp"), + ] + ), + GameLib( + "Overlay 110", + [ + Object("110_PlayerGet/PlayerGet.cpp"), + ] + ), + LibC( + "libc", + [ + Object("abort_exit_arm_eabi.c"), + Object("ansi_files.c"), + Object("float.c"), + Object("locale.c"), + Object("arith.c"), + Object("buffer_io.c"), + Object("file_io.c"), + Object("math_api.c"), + Object("mbstring.c"), + Object("mem.c"), + Object("mem_funcs.c"), + Object("secure_error.c"), + Object("signal.c"), + Object("string.c"), + Object("wmem.c"), + Object("wprintf.c"), + Object("wstring.c"), + Object("ansi_fp.c"), + Object("extras.c"), + Object("math/e_log.c"), + Object("math/e_log10.c"), + Object("math/e_pow.c"), + Object("math/s_ceil.c"), + Object("math/s_copysign.c"), + Object("math/s_fabs.c"), + Object("math/s_frexp.c"), + Object("math/s_ldexp.c"), + Object("math/w_log.c"), + Object("math/w_log10f.c"), + Object("math/w_pow.c"), + ] + ), + LibCPP( + "libcpp", + [ + Object("__register_global_object.c") + ] + ), +] def main(): - if platform is None: - return - - delinks_json = None - can_run_dsd = check_can_run_dsd() - if can_run_dsd: - out = subprocess.run([ - DSD, - "--force-color", - "json", - "delinks", - "--config-path", config_path / args.version / "arm9" / "config.yaml" - ], capture_output=True, text=True) - if out.returncode != 0: - print(f"Error running dsd:\n{out.stderr.strip()}") - return - delinks_json = json.loads(out.stdout) - - project = Project(args.version, platform=platform, delinks_json=delinks_json) - - - with build_ninja_path.open("w") as file: - n = ninja_syntax.Writer(file) - - n.rule( - name="download_tool", - command=f'{PYTHON} tools/download_tool.py $tool $tag --path $path' - ) - n.newline() - - if arm7_bios_path.is_file(): - n.variable("arm7_bios_flag", f"--arm7-bios {arm7_bios_path.relative_to(root_path)}") - else: - n.variable("arm7_bios_flag", "") - n.newline() - - n.rule( - name="extract", - command=f"{DSD} {DSD_BASE_FLAGS} rom extract --rom $in --output-path $output_path $arm7_bios_flag" - ) - n.newline() - - n.rule( - name="delink", - command=f"{DSD} {DSD_BASE_FLAGS} delink --config-path $config_path" - ) - n.newline() - - n.rule( - name="disassemble", - command=f"{DSD} {DSD_BASE_FLAGS} dis --config-path $config_path --asm-path $output_path --ual" - ) - n.newline() - - # -MMD excludes all includes instead of just system includes for some reason, so use -MD instead. - mwcc_cmd = f'{WINE} "{CC}" {CC_FLAGS} {CC_INCLUDES} $cc_flags -DVERSION=$game_version -MD -c $in -o $basedir' - mwcc_implicit = [CC] - if platform.system != "windows": - transform_dep = "tools/transform_dep.py" - mwcc_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" - mwcc_implicit.append(transform_dep) - if WINE == DEFAULT_WIBO_PATH: - mwcc_implicit.append(WINE) - n.rule( - name="mwcc", - command=mwcc_cmd, - depfile="$basefile.d", - ) - n.newline() - - n.rule( - name="lcf", - command=f"{DSD} {DSD_BASE_FLAGS} lcf -c $config_path" - ) - n.newline() - - n.rule( - name="mwld", - command=f'{WINE} "{LD}" {LD_FLAGS} $extra_ld_flags @$objects_file $lcf_file -o $out' - ) - n.newline() - - n.rule( - name="rom_config", - command=f"{DSD} {DSD_BASE_FLAGS} rom config --elf $in --config $config_path" - ) - n.newline() - - n.rule( - name="rom_build", - command=f"{DSD} {DSD_BASE_FLAGS} rom build --config $in --rom $out $arm7_bios_flag" - ) - n.newline() - - n.rule( - name="objdiff", - command=f"{DSD} {DSD_BASE_FLAGS} objdiff --config-path $config_path {DSD_OBJDIFF_ARGS}" - ) - n.newline() - - n.rule( - name="objdiff_report", - command=f"{OBJDIFF} report generate -o $out" - ) - n.newline() - - n.rule( - name="m2ctx", - command=f"{PYTHON} tools/m2ctx.py -f $out $in" - ) - n.newline() - - n.rule( - name="check_modules", - command=f"{DSD} {DSD_BASE_FLAGS} check modules --config-path $config_path --fail" - ) - n.newline() - - n.rule( - name="check_symbols", - command=f"{DSD} {DSD_BASE_FLAGS} check symbols --config-path $config_path --elf-path $elf_path --fail --max-lines 20" - ) - n.newline() - - n.rule( - name="apply", - command=f"{DSD} {DSD_BASE_FLAGS} apply --config-path $config_path --elf-path $elf_path" - ) - n.newline() - - n.rule( - name="sha1", - command=f"{PYTHON} tools/sha1.py $in -c $sha1_file" - ) - n.newline() - - configure_cmdline = subprocess.list2cmdline(sys.argv[1:]) - n.rule( - name="configure", - command=f"{PYTHON} tools/configure.py {configure_cmdline}", - generator=True - ) - n.newline() - - n.rule( - name="format_delinks", - command=f'{PYTHON} tools/format_delinks.py' - ) - n.newline() - - add_download_tool_builds(n, project) - add_configure_build(n, project) - - if can_run_dsd: - add_extract_build(n, project) - add_delink_and_lcf_builds(n, project) - add_disassemble_builds(n, project) - add_mwcc_builds(n, project, mwcc_implicit) - add_mwld_and_rom_builds(n, project) - add_check_builds(n, project) - add_objdiff_builds(n, project) - add_apply_build(n, project) - - n.build( - rule="format_delinks", - outputs="format_delinks" - ) - n.newline() - - n.default(["format_delinks", "objdiff", "check", "sha1"]) - else: - n.default(["download_tools"]) - - -def add_download_tool_builds(n: ninja_syntax.Writer, project: Project): - downloads: list[str] = [] - - if args.dsd is None: - downloads.append(DSD) - n.build( - rule="download_tool", - outputs=DSD, - variables={ - "tool": "dsd", - "tag": DSD_VERSION, - "path": DSD, - }, - ) - n.newline() - - downloads.append(OBJDIFF) - n.build( - rule="download_tool", - outputs=OBJDIFF, - variables={ - "tool": "objdiff", - "tag": OBJDIFF_VERSION, - "path": OBJDIFF, - } - ) - n.newline() - - if args.compiler is None: - downloads.extend([CC, LD]) - n.build( - rule="download_tool", - outputs=[CC, LD], - variables={ - "tool": "mwccarm", - "tag": "latest", - "path": str(tools_path), - }, - ) - n.newline() - - if project.platform.system != "windows" and WINE == DEFAULT_WIBO_PATH: - downloads.append(WINE) - n.build( - rule="download_tool", - outputs=WINE, - variables={ - "tool": "wibo", - "tag": WIBO_VERSION, - "path": WINE, - }, - ) - n.newline() - - n.build( - inputs=downloads, - rule="phony", - outputs="download_tools", - ) - n.newline() - - -def add_extract_build(n: ninja_syntax.Writer, project: Project): - if not args.no_extract: - n.build( - inputs=str(project.baserom()), - implicit=DSD, - rule="extract", - outputs=str(project.baserom_config()), - variables={ - "output_path": str(project.game_extract) - } - ) - n.newline() - - -def add_mwld_and_rom_builds(n: ninja_syntax.Writer, project: Project): - n.comment("Run linker") - objects_to_link = [file['object_to_link'] for file in project.files()] - elf_file = str(project.arm9_o()) - lcf_file = project.arm9_lcf_file() - objects_file = project.arm9_objects_file() - if len(objects_to_link) > 0: - ld_implicit = [LD] - if WINE == DEFAULT_WIBO_PATH: - ld_implicit.append(WINE) - n.build( - inputs=[*objects_to_link, lcf_file, objects_file], - implicit=ld_implicit, - rule="mwld", - outputs=elf_file, - variables={ - 'extra_ld_flags': ARM9_LD_FLAGS, - 'lcf_file': str(lcf_file), - 'objects_file': str(objects_file), - } - ) - n.newline() - - n.build( - inputs=elf_file, - rule="phony", - outputs="arm9", - ) - n.newline() - - rom_config_file = str(project.build_rom_config()) - n.build( - inputs=elf_file, - implicit=DSD, - rule="rom_config", - outputs=rom_config_file, - variables={ - "config_path": str(project.arm9_config_yaml()), - } - ) - n.newline() - - rom_file = project.build_rom() - n.build( - inputs=rom_config_file, - implicit=DSD, - rule="rom_build", - outputs=rom_file, - ) - n.newline() - - n.build( - inputs=rom_file, - rule="phony", - outputs="rom", - ) - n.newline() - - n.build( - inputs=rom_file, - rule="sha1", - variables={ - "sha1_file": str(Path(rom_file).with_suffix(".sha1")) - }, - outputs="sha1", - ) - n.newline() - - -def add_mwcc_builds(n: ninja_syntax.Writer, project: Project, mwcc_implicit: list[str]): - for source_file in get_c_cpp_files([src_path, libs_path]): - src_obj_path = project.game_build / source_file - cc_flags: list[str] = [] - if is_cpp(source_file): - cc_flags.append("-lang=c++") - elif is_c(source_file): - cc_flags.append("-lang=c") - n.build( - inputs=str(source_file), - implicit=mwcc_implicit, - rule="mwcc", - outputs=str(src_obj_path.with_suffix(".o")), - variables={ - "game_version": project.game_version.upper(), - "cc_flags": " ".join(cc_flags), - "basedir": os.path.dirname(src_obj_path), - "basefile": str(src_obj_path.with_suffix("")), - }, - ) - n.newline() - - extension = source_file.suffix - ctx_file = str(src_obj_path.with_suffix(f".ctx{extension}")) - n.build( - inputs=str(source_file), - rule="m2ctx", - outputs=ctx_file, - ) - n.newline() - - -def get_c_cpp_files(dirs: list[Path]): - for dir in dirs: - for root, _, files in os.walk(dir): - root = Path(root) - for file in files: - if is_cpp(file) or is_c(file): - yield root / file - - -def is_cpp(name: str | Path): - return Path(name).suffix in [".cpp"] - - -def is_c(name: str | Path): - return Path(name).suffix in [".c"] - - -def add_delink_and_lcf_builds(n: ninja_syntax.Writer, project: Project): - rom_config = str(project.baserom_config()) - delink_files = project.delink_files() - if len(delink_files) > 0: - n.comment("Delink ELF binaries when any delinks.txt file is modified") - n.build( - inputs=project.dsd_configs() + [rom_config], - implicit=DSD, - rule="delink", - outputs=delink_files, - variables={ - "config_path": str(project.arm9_config_yaml()), - } - ) - n.newline() - - n.build( - inputs=delink_files, - rule="phony", - outputs="delink" - ) - n.newline() - - lcf_file = project.arm9_lcf_file() - objects_file = project.arm9_objects_file() - n.build( - inputs=project.delinks_files + [str(rom_config)], - implicit=DSD, - rule="lcf", - outputs=[lcf_file, objects_file], - variables={ - "config_path": str(project.arm9_config_yaml()), - } - ) - n.newline() - - -def add_disassemble_builds(n: ninja_syntax.Writer, project: Project): - n.build( - inputs=project.dsd_configs(), - implicit=DSD, - rule="disassemble", - outputs="dis", - variables={ - "config_path": str(project.arm9_config_yaml()), - "output_path": str(project.arm9_disassembly_dir()), - } - ) - n.newline() - - -def add_check_builds(n: ninja_syntax.Writer, project: Project): - n.build( - inputs=str(project.arm9_o()), - rule="check_modules", - outputs="check_modules", - variables={ - "config_path": str(project.arm9_config_yaml()), - }, - ) - n.newline() - - n.build( - inputs=str(project.arm9_o()), - rule="check_symbols", - outputs="check_symbols", - variables={ - "config_path": str(project.arm9_config_yaml()), - "elf_path": str(project.arm9_o()), - }, - ) - n.newline() - - n.build( - inputs=["check_modules", "check_symbols"], - rule="phony", - outputs="check", - ) - n.newline() - - -def add_objdiff_builds(n: ninja_syntax.Writer, project: Project): - n.build( - inputs=project.dsd_configs(), - implicit=DSD, - rule="objdiff", - outputs="objdiff.json", - variables={ - "config_path": str(project.arm9_config_yaml()), - } - ) - n.newline() - - n.build( - inputs="objdiff.json", - rule="phony", - outputs="objdiff", - ) - n.newline() - - delink_files = project.delink_files() - n.build( - inputs=["objdiff.json"], - implicit=[OBJDIFF] + delink_files + project.source_object_files(), - rule="objdiff_report", - outputs=str(project.objdiff_report()), - ) - n.newline() - - n.build( - inputs=str(project.objdiff_report()), - rule="phony", - outputs="report", - ) - n.newline() - - -def add_configure_build(n: ninja_syntax.Writer, project: Project): - this_file = str(Path(__file__).resolve()) - n.build( - outputs="build.ninja", - rule="configure", - implicit=[ - this_file, - # Require dsd to exist when rerunning configure.py - DSD, - *project.dsd_configs(), - ] - ) - - -def add_apply_build(n: ninja_syntax.Writer, project: Project): - n.build( - inputs=project.dsd_configs() + [str(project.arm9_o())], - implicit=DSD, - rule="apply", - outputs="apply", - variables={ - "config_path": str(project.arm9_config_yaml()), - "elf_path": str(project.arm9_o()), - } - ) - n.newline() - - -def get_config_files(game_config: Path, name: str) -> list[str]: - return [ - f"{root}/{file}" - for root, _, files in os.walk(game_config) - for file in files - if file == name - ] + for version in config.game_versions: + config.delinks_files[version] = config.get_config_files(version, "delinks.txt") + config.relocs_files[version] = config.get_config_files(version, "relocs.txt") + config.symbols_files[version] = config.get_config_files(version, "symbols.txt") + + if config.check_can_run_dsd(): + out = subprocess.run([ + str(config.dsd_path), + "--force-color", + "json", + "delinks", + "--config-path", config.config_path / version / "arm9" / "config.yaml" + ], capture_output=True, text=True) + assert out.returncode == 0, f"Error running dsd:\n{out.stderr.strip()}" + + config.delinks_jsons[version] = json.loads(out.stdout) + + process_project(config, args) if __name__ == "__main__": diff --git a/tools/m2ctx.py b/tools/m2ctx.py index 158ce517..3ae0c237 100755 --- a/tools/m2ctx.py +++ b/tools/m2ctx.py @@ -15,6 +15,7 @@ parser.add_argument('-f', type=str, dest='out_file', required=False, help='Outpu parser.add_argument('-c', action=argparse.BooleanOptionalAction, dest='clipboard', required=False, help='Copy output to clipboard') parser.add_argument('-e', type=str, dest='encoding', required=False, default="utf-8", help='Input file encoding') parser.add_argument('-v', action=argparse.BooleanOptionalAction, dest='verbose', required=False, help='Verbose error output') +parser.add_argument('-g', dest="version", required=False, default="EUR") args = parser.parse_args() CXX_FLAGS = [ @@ -23,6 +24,7 @@ CXX_FLAGS = [ '-Ilibs/c/include', '-Ilibs/cpp/include', '-Ilibs/runtime/include', + f'-DVERSION={args.version}', ] script_dir = Path(os.path.dirname(os.path.realpath(__file__))) diff --git a/tools/objdiff_config.py b/tools/objdiff_config.py new file mode 100644 index 00000000..f0bd20a2 --- /dev/null +++ b/tools/objdiff_config.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 + +import json +import copy + +from pathlib import Path +from typing import Optional + + +class ConfigUnit: + def __init__(self, name: str, options: dict[str, str]): + self.name = name + self.cflags = options["cflags"] + self.extra_cflags = options["extra_cflags"] + self.mw_version = options["mw_version"] + + def get_all_cflags(self): + return " ".join(self.cflags) + " " + " ".join(self.extra_cflags) + + +class ConfigVersion: + def __init__(self, name: Optional[str], options: dict[str, str], units: list[ConfigUnit], root_path: Path): + self.name = name + self.objdiff_path = Path(options["objdiff"]).resolve() + + objdiff_json = json.loads(self.objdiff_path.read_text()) + self.objdiff_json: dict = copy.copy(objdiff_json) + + # deprecated options + self.objdiff_json.pop("target_dir") + self.objdiff_json.pop("base_dir") + self.objdiff_json.pop("build_base") + + for i, unit_dict in enumerate(objdiff_json["units"]): + if "name" in unit_dict and self.name is not None: + unit_dict["name"] = f"{self.name}/{unit_dict['name']}" + + def get_cleaned_path(base_path: Path): + parts = list(base_path.parts) + for part in base_path.parts: + if "build" in part: + break + parts.remove(part) + return Path("/".join(parts)) + + if "target_path" in unit_dict: + target_path = root_path / get_cleaned_path(Path(unit_dict["target_path"])) + self.objdiff_json["units"][i]["target_path"] = str(target_path.resolve()) + + if "base_path" in unit_dict: + base_path = root_path / get_cleaned_path(Path(unit_dict["base_path"])) + self.objdiff_json["units"][i]["base_path"] = str(base_path) + + if "scratch" in unit_dict: + ctx_path = get_cleaned_path(Path(unit_dict["scratch"]["ctx_path"])) + self.objdiff_json["units"][i]["scratch"]["ctx_path"] = str(ctx_path) + + entry = self.get_entry_from_name(units, self.objdiff_json["units"][i]["name"]) + if entry is not None: + self.objdiff_json["units"][i]["scratch"]["c_flags"] = entry.get_all_cflags() + self.objdiff_json["units"][i]["scratch"]["compiler"] = entry.mw_version + + def get_entry_from_name(self, units: list[ConfigUnit], base_name: str): + split = base_name.split("/") + split.pop(0) + + name = Path(base_name).stem + for entry in units: + if name in entry.name: + return entry + + return None + +class ObjdiffConfig: + def __init__(self, root_path: Path): + self.root_path = root_path + self.versions: list[ConfigVersion] = [] + + def get_json(self): + assert len(self.versions) > 0, "no versions?" + out_json = copy.copy(self.versions[0].objdiff_json) + + # merge units + for i, entry in enumerate(self.versions): + if i > 0: + out_json["units"].extend(entry.objdiff_json["units"]) + + return out_json + + @staticmethod + def new(path: str): + json_path = Path(path).resolve() + cfg_json = json.loads(json_path.read_text()) + cfg = ObjdiffConfig(Path(cfg_json["root_path"])) + + units: list[ConfigUnit] = [] + for name, options in cfg_json["units"].items(): + units.append(ConfigUnit(name, options)) + + multi_version = len(cfg_json["versions"].items()) > 1 + for name, options in cfg_json["versions"].items(): + cfg.versions.append(ConfigVersion(name if multi_version else None, options, units, cfg.root_path)) + + return cfg + + +def main(): + cfg = ObjdiffConfig.new("build/objdiff_cfg.json") + + out_json = cfg.get_json() + out_path = Path("objdiff.json").resolve() + with out_path.open("w") as f: + json.dump(out_json, f, indent=2) + + +if __name__ == "__main__": + main() diff --git a/tools/project.py b/tools/project.py new file mode 100644 index 00000000..30b4f37b --- /dev/null +++ b/tools/project.py @@ -0,0 +1,920 @@ +import os +import sys +import json +import subprocess +import ninja_syntax + +from pathlib import Path +from typing import Any, Optional, Dict, List +from get_platform import get_platform + + +COMPILER_MAP = { + "1.2/base": "mwcc_20_72", + "1.2/sp2": "mwcc_20_79", + "1.2/sp2p3": "mwcc_20_82", + "1.2/sp3": "mwcc_20_84", + "1.2/sp4": "mwcc_20_87", + "2.0/base": "mwcc_30_114", + "2.0/sp1": "mwcc_30_123", + "2.0/sp1p2": "mwcc_30_126", + "2.0/sp1p5": "mwcc_30_131", + "2.0/sp1p6": "mwcc_30_133", + "2.0/sp1p7": "mwcc_30_134", + "2.0/sp2": "mwcc_30_136", + "2.0/sp2p2": "mwcc_30_137", + "2.0/sp2p3": "mwcc_30_138", + "2.0/sp2p4": "mwcc_30_139", + "dsi/1.1": "mwcc_40_1018", + "dsi/1.1p1": "mwcc_40_1024", + "dsi/1.2": "mwcc_40_1026", + "dsi/1.2p1": "mwcc_40_1027", + "dsi/1.2p2": "mwcc_40_1028", + "dsi/1.3": "mwcc_40_1034", + "dsi/1.3p1": "mwcc_40_1036", + "dsi/1.6sp1": "mwcc_40_1051", + "dsi/1.6sp2": "mwcc_40_1051", +} + +Library = Dict[str, Any] + +def get_c_cpp_files(dirs: list[Path]): + for dir in dirs: + for root, _, files in os.walk(dir): + root = Path(root) + for file in files: + if is_cpp(file) or is_c(file): + yield root / file + + +def is_cpp(name: str | Path): + return Path(name).suffix in [".cpp"] + + +def is_c(name: str | Path): + return Path(name).suffix in [".c"] + +class Object: + def __init__(self, name: str, **options: Any): + self.name = name + + self.options: Dict[str, Any] = { + "source": name, + "mw_version": None, + "asflags": None, + "extra_asflags": [], + "cflags": None, + "extra_cflags": [], + "asm_dir": None, + "src_dir": None, + } + self.options.update(options) + + self.src_path: Optional[Path] = None + self.asm_path: Optional[Path] = None + self.src_obj_path: Optional[Path] = None + self.asm_obj_path: Optional[Path] = None + self.ctx_path: Optional[Path] = None + + def resolve(self, config: "ProjectConfig", lib: Library): + # Use object options, then library options + obj = Object(self.name, **lib) + + for key, value in self.options.items(): + if value is not None or key not in obj.options: + obj.options[key] = value + + # Use default options from config + def set_default(key: str, value: Any) -> None: + if obj.options[key] is None: + obj.options[key] = value + + set_default("asflags", config.asflags) + set_default("mw_version", config.mwcc_version) + set_default("asm_dir", config.asm_path) + set_default("src_dir", config.src_path) + + # Resolve paths + obj.src_path = Path(obj.options["src_dir"]) / obj.options["source"] + if obj.options["asm_dir"] is not None: + path: Path = Path(obj.options["asm_dir"]) / obj.options["source"] + obj.asm_path = path.with_suffix(".s") + base_name = Path(self.name).with_suffix("") + obj.src_obj_path = config.build_path / "src" / f"{base_name}.o" + obj.asm_obj_path = config.build_path / "mod" / f"{base_name}.o" + obj.ctx_path = config.build_path / "src" / f"{base_name}.ctx" + return obj + + +class ProjectConfig: + def __init__(self, game: str, mwcc_root: Optional[str], mwcc_tag: str, wine: str, dsd: Optional[str], cfg_script: Path): + # Tools + self.dsd_tag: Optional[str] = None + self.wibo_tag: Optional[str] = None + self.objdiff_tag: Optional[str] = None + self.mwcc_tag = mwcc_tag + self.mwcc_root = Path(mwcc_root).resolve() if mwcc_root is not None else self.tools_path / "mwccarm" + self.cfg_script = cfg_script + + # Platform info + self.platform = get_platform() + """Host platform information""" + + assert self.platform is not None + self.default_wibo_path = "./wibo" + self.wine_path = wine if self.platform.system != "windows" else "" + self.dsd_path = (Path(dsd) if dsd is not None else (self.root_path / f"dsd{self.platform.exe}")).resolve() + self.objdiff_path = (self.root_path / f"objdiff-cli{self.platform.exe}").resolve() + self.cc_path = (self.mwcc_path / "mwccarm.exe").resolve() + self.ld_path = (self.mwcc_path / "mwldarm.exe").resolve() + self.python_path = Path(sys.executable).resolve() + + self.dsd_base_flags = [ + "--force-color", # Force color output + ] + + self.game = game + """Name of the game""" + + self.game_versions: list[str] = [] + """Versions of the game""" + + self.delinks_jsons: dict[str, Optional[Any]] = {} + """Delinks JSON data from dsd (version: json)""" + + self.delinks_files: dict[str, list[str]] = {} + """Paths to every delinks.txt file""" + + self.relocs_files: dict[str, list[str]] = {} + """Paths to every relocs.txt file""" + + self.symbols_files: dict[str, list[str]] = {} + """Paths to every symbols.txt file""" + + self.libs: Optional[List[Library]] = None + """List of libraries""" + + self.asflags: Optional[List[str]] = None + """Assembler flags""" + + self.cflags_base: Optional[List[str]] = None + """Base C flags""" + + self.ldflags: Optional[List[str]] = None + """Base C flags""" + + includes = [self.root_path / "include"] + for root, dirs, _ in os.walk(self.libs_path): + for dir in dirs: + if dir == "include": + includes.append(Path(root) / dir) + + self.includes = " ".join(f"-i {include}" for include in includes) + """C/C++ includes""" + + self.auto_add_sources: bool = False + """Adds rules for files missing from the libs list (with the base cflags as the default)""" + + self.warn_missing_source: bool = True + """Warn on missing source file""" + + def get_game_config(self, version: str): + """Root directory for dsd configs""" + config_path = self.config_path / version + assert config_path.is_dir(), f"Version '{version}' not recognized" + return config_path + + def get_game_build(self, version: str): + """Path to build directory""" + return self.build_path / version + + def get_game_extract(self, version: str): + """Path to extract directory""" + return self.extract_path / version + + @property + def dsd_version(self): + assert self.dsd_tag is not None + return self.dsd_tag + + @property + def wibo_version(self): + assert self.wibo_tag is not None + return self.wibo_tag + + @property + def objdiff_version(self): + assert self.objdiff_tag is not None + return self.objdiff_tag + + @property + def mwcc_version(self): + assert self.mwcc_tag is not None + return self.mwcc_tag + + @property + def current_path(self): + return Path(__name__) + + @property + def root_path(self): + return self.current_path.parent + + @property + def build_ninja_path(self): + return self.root_path / "build.ninja" + + @property + def arm7_bios_path(self): + return self.root_path / "arm7_bios.bin" + + @property + def config_path(self): + return self.root_path / "config" + + @property + def build_path(self): + return self.root_path / "build" + + @property + def asm_path(self): + return self.root_path / "asm" + + @property + def src_path(self): + return self.root_path / "src" + + @property + def libs_path(self): + return self.root_path / "libs" + + @property + def extract_path(self): + return self.root_path / "extract" + + @property + def tools_path(self): + return self.root_path / "tools" + + @property + def mwcc_path(self): + return self.mwcc_root / self.mwcc_version + + @property + def dsd_flags(self): + return " ".join(self.dsd_base_flags) + + def dsd_configs(self, version: str) -> list[str]: + return self.delinks_files[version] + self.relocs_files[version] + self.symbols_files[version] + + def arm9_config_yaml(self, version: str) -> Path: + return self.get_game_config(version) / "arm9" / "config.yaml" + + def baserom(self, version: str) -> Path: + return self.extract_path / f'baserom_{self.game}_{version}.nds' + + def build_rom(self, version: str) -> str: + return f"{self.game}_{version}.nds" + + def baserom_config(self, version: str,) -> Path: + return self.get_game_extract(version) / 'config.yaml' + + def build_rom_config(self, version: str) -> Path: + return self.get_game_build(version) / "build" / "rom_config.yaml" + + def source_object_files(self, version: str) -> list[str]: + files: list[str] = [] + for source_file in get_c_cpp_files([self.src_path, self.libs_path]): + src_obj_path = self.get_game_build(version) / source_file + files.append(str(src_obj_path.with_suffix(".o"))) + return files + + def arm9_o(self, version: str) -> Path: + return self.get_game_build(version) / "arm9.o" + + def arm9_disassembly_dir(self, version: str) -> Path: + return self.get_game_build(version) / "asm" + + def objdiff_report(self, version: str) -> Path: + return f"report_{version}.json" + + def files(self, version: str) -> list[dict[str, str]]: + if self.delinks_jsons[version] is None: + return [] + return self.delinks_jsons[version]['files'] + + def delink_files(self, version: str) -> list[str]: + delink_files = [file['delink_file'] for file in self.files(version)] + return list(set(delink_files)) + + def arm9_lcf_file(self, version: str) -> str: + if self.delinks_jsons[version] is None: + return "" + return self.delinks_jsons[version]['arm9_lcf_file'] + + def arm9_objects_file(self, version: str) -> str: + if self.delinks_jsons[version] is None: + return "" + return self.delinks_jsons[version]['arm9_objects_file'] + + def get_config_files(self, version: str, name: str) -> list[str]: + return [ + f"{root}/{file}" + for root, _, files in os.walk(self.get_game_config(version)) + for file in files + if file == name + ] + + def get_decompme_compiler(self): + return COMPILER_MAP[self.mwcc_version] + + # Creates a map of object names to Object instances + # Options are fully resolved from the library and object + def objects(self) -> Dict[str, Object]: + out = {} + for lib in self.libs or {}: + objects: List[Object] = lib["objects"] + for obj in objects: + if obj.name in out: + sys.exit(f"Duplicate object name {obj.name}") + out[obj.name] = obj.resolve(self, lib) + return out + + def check_can_run_dsd(self) -> bool: + try: + output = subprocess.run([str(self.dsd_path), "--version"], capture_output=True, text=True, check=True) + version = output.stdout.strip().split(" ")[-1] + if not version.startswith("v"): + version = "v" + version + + # If it's not the correct version, Ninja will download it and then rerun this script + return version == self.dsd_version + except subprocess.CalledProcessError: + return False + except FileNotFoundError: + return False + + +def add_download_tool_builds(cfg: ProjectConfig, n: ninja_syntax.Writer, args: Any): + downloads: list[str] = [] + + if args.dsd is None: + downloads.append(str(cfg.dsd_path)) + n.build( + rule="download_tool", + outputs=str(cfg.dsd_path), + variables={ + "tool": "dsd", + "tag": cfg.dsd_version, + "path": cfg.dsd_path, + }, + ) + n.newline() + + downloads.append(str(cfg.objdiff_path)) + n.build( + rule="download_tool", + outputs=str(cfg.objdiff_path), + variables={ + "tool": "objdiff", + "tag": cfg.objdiff_version, + "path": cfg.objdiff_path, + } + ) + n.newline() + + if args.compiler is None: + downloads.extend([str(cfg.cc_path), str(cfg.ld_path)]) + n.build( + rule="download_tool", + outputs=[str(cfg.cc_path), str(cfg.ld_path)], + variables={ + "tool": "mwccarm", + "tag": "latest", + "path": str(cfg.tools_path), + }, + ) + n.newline() + + if cfg.platform.system != "windows" and cfg.wine_path == cfg.default_wibo_path: + downloads.append(str(cfg.wine_path)) + n.build( + rule="download_tool", + outputs=cfg.wine_path, + variables={ + "tool": "wibo", + "tag": cfg.wibo_version, + "path": cfg.wine_path, + }, + ) + n.newline() + + n.build( + inputs=downloads, + rule="phony", + outputs="download_tools", + ) + n.newline() + + +def add_configure_build(cfg: ProjectConfig, n: ninja_syntax.Writer): + dsd_cfg = [] + + for version in cfg.game_versions: + dsd_cfg.extend(cfg.dsd_configs(version)) + + n.build( + outputs="build.ninja", + rule="configure", + implicit=[ + str(cfg.cfg_script), + # Require dsd to exist when rerunning configure.py + str(cfg.dsd_path), + *dsd_cfg, + ] + ) + + +def add_extract_build(cfg: ProjectConfig, version: str, n: ninja_syntax.Writer, args: Any): + if not args.no_extract: + n.build( + inputs=str(cfg.baserom(version)), + implicit=str(cfg.dsd_path), + rule="extract", + outputs=str(cfg.baserom_config(version)), + variables={ + "output_path": str(cfg.get_game_extract(version)) + } + ) + n.newline() + + +def add_delink_and_lcf_builds(cfg: ProjectConfig, version: str, n: ninja_syntax.Writer): + rom_config = str(cfg.baserom_config(version)) + delink_files = cfg.delink_files(version) + if len(delink_files) > 0: + n.comment("Delink ELF binaries when any delinks.txt file is modified") + n.build( + inputs=cfg.dsd_configs(version) + [rom_config], + implicit=str(cfg.dsd_path), + rule="delink", + outputs=delink_files, + variables={ + "config_path": str(cfg.arm9_config_yaml(version)), + } + ) + n.newline() + + n.build( + inputs=delink_files, + rule="phony", + outputs=f"delink_{version}" + ) + n.newline() + + lcf_file = cfg.arm9_lcf_file(version) + objects_file = cfg.arm9_objects_file(version) + n.build( + inputs=cfg.delinks_files[version] + [str(rom_config)], + implicit=str(cfg.dsd_path), + rule="lcf", + outputs=[lcf_file, objects_file], + variables={ + "config_path": str(cfg.arm9_config_yaml(version)), + } + ) + n.newline() + + +def add_disassemble_builds(cfg: ProjectConfig, version: str, n: ninja_syntax.Writer): + n.build( + inputs=cfg.dsd_configs(version), + implicit=str(cfg.dsd_path), + rule="disassemble", + outputs=f"dis_{version}", + variables={ + "config_path": str(cfg.arm9_config_yaml(version)), + "output_path": str(cfg.arm9_disassembly_dir(version)), + } + ) + n.newline() + + +def add_mwcc_builds(cfg: ProjectConfig, version: str, objects: Dict[str, Object], n: ninja_syntax.Writer, mwcc_implicit: list[str]): + file_map: dict[str, list[str]] = {} + + for object in objects.values(): + file_map[str(object.src_path)] = object.options["cflags"] + object.options["extra_cflags"] + + if cfg.auto_add_sources: + for source_file in get_c_cpp_files([cfg.src_path, cfg.libs_path]): + if str(source_file) not in file_map: + file_map[str(source_file)] = cfg.cflags_base + + for src_file, cc_flags in file_map.items(): + source_file = Path(src_file) + src_obj_path = cfg.get_game_build(version) / source_file + + if cfg.warn_missing_source and not source_file.exists(): + print(f"WARNING: path not found for `{source_file}`") + + if "-lang=c++" not in cc_flags or "-lang=c" not in cc_flags: + if is_cpp(source_file): + cc_flags.append("-lang=c++") + elif is_c(source_file): + cc_flags.append("-lang=c") + + n.build( + inputs=str(source_file), + implicit=mwcc_implicit, + rule="mwcc", + outputs=str(src_obj_path.with_suffix(".o")), + variables={ + "game_version": version.upper(), + "cc_flags": " ".join(cc_flags), + "basedir": str(src_obj_path.parent), + "basefile": str(src_obj_path.with_suffix("")), + }, + ) + n.newline() + + extension = source_file.suffix + ctx_file = str(src_obj_path.with_suffix(f".ctx{extension}")) + n.build( + inputs=str(source_file), + rule="m2ctx", + outputs=ctx_file, + variables={ + "version": version.upper(), + }, + ) + n.newline() + + +def add_mwld_and_rom_builds(cfg: ProjectConfig, version: str, n: ninja_syntax.Writer): + n.comment("Run linker") + objects_to_link = [file['object_to_link'] for file in cfg.files(version)] + elf_file = str(cfg.arm9_o(version)) + lcf_file = cfg.arm9_lcf_file(version) + objects_file = cfg.arm9_objects_file(version) + if len(objects_to_link) > 0: + ld_implicit = [str(cfg.ld_path)] + if cfg.wine_path == cfg.default_wibo_path: + ld_implicit.append(cfg.wine_path) + + # Only passed to the final arm9.o link + arm9_ld_flags = " ".join([ + "-m Entry", # Set entry function + ]) + n.build( + inputs=[*objects_to_link, lcf_file, objects_file], + implicit=ld_implicit, + rule="mwld", + outputs=elf_file, + variables={ + 'extra_ld_flags': arm9_ld_flags, + 'lcf_file': str(lcf_file), + 'objects_file': str(objects_file), + } + ) + n.newline() + + n.build( + inputs=elf_file, + rule="phony", + outputs=f"arm9_{version}", + ) + n.newline() + + rom_config_file = str(cfg.build_rom_config(version)) + n.build( + inputs=elf_file, + implicit=str(cfg.dsd_path), + rule="rom_config", + outputs=rom_config_file, + variables={ + "config_path": str(cfg.arm9_config_yaml(version)), + } + ) + n.newline() + + rom_file = cfg.build_rom(version) + n.build( + inputs=rom_config_file, + implicit=str(cfg.dsd_path), + rule="rom_build", + outputs=rom_file, + ) + n.newline() + + n.build( + inputs=rom_file, + rule="phony", + outputs=f"rom_{version}", + ) + n.newline() + + n.build( + inputs=rom_file, + rule="sha1", + variables={ + "sha1_file": str(Path(rom_file).with_suffix(".sha1")) + }, + outputs=f"sha1_{version}", + ) + n.newline() + + +def add_check_builds(cfg: ProjectConfig, version: str, n: ninja_syntax.Writer): + n.build( + inputs=str(cfg.arm9_o(version)), + rule="check_modules", + outputs=f"check_modules_{version}", + variables={ + "config_path": str(cfg.arm9_config_yaml(version)), + }, + ) + n.newline() + + n.build( + inputs=str(cfg.arm9_o(version)), + rule="check_symbols", + outputs=f"check_symbols_{version}", + variables={ + "config_path": str(cfg.arm9_config_yaml(version)), + "elf_path": str(cfg.arm9_o(version)), + }, + ) + n.newline() + + n.build( + inputs=[f"check_modules_{version}", f"check_symbols_{version}"], + rule="phony", + outputs=f"check_{version}", + ) + n.newline() + + +def add_objdiff_builds(cfg: ProjectConfig, version: str, n: ninja_syntax.Writer): + out_path = cfg.get_game_build(version) / "objdiff.json" + + n.build( + inputs=cfg.dsd_configs(version), + implicit=str(cfg.dsd_path), + rule="objdiff", + outputs=str(out_path), + variables={ + "config_path": str(cfg.arm9_config_yaml(version)), + "out_path": str(cfg.get_game_build(version)), + } + ) + n.newline() + + n.build( + inputs=str(out_path), + rule="phony", + outputs=f"objdiff_{version}.json", + ) + n.newline() + + delink_files = cfg.delink_files(version) + n.build( + inputs=[str(out_path)], + implicit=[str(cfg.objdiff_path)] + delink_files + cfg.source_object_files(version), + rule="objdiff_report", + outputs=str(cfg.objdiff_report(version)), + variables={ + "dir": str(cfg.get_game_build(version)), + "filename": "report.json" + } + ) + n.newline() + + n.build( + inputs=str(cfg.objdiff_report(version)), + rule="phony", + outputs=f"report_{version}", + ) + n.newline() + + +def add_apply_build(cfg: ProjectConfig, version: str, n: ninja_syntax.Writer): + n.build( + inputs=cfg.dsd_configs(version) + [str(cfg.arm9_o(version))], + implicit=str(cfg.dsd_path), + rule="apply", + outputs=f"apply_{version}", + variables={ + "config_path": str(cfg.arm9_config_yaml(version)), + "elf_path": str(cfg.arm9_o(version)), + } + ) + n.newline() + + +def create_objdiff_fixup_config(cfg: ProjectConfig, objects: Dict[str, Object]): + out_json = {} + + out_json["root_path"] = str(cfg.root_path.resolve()) + out_json["versions"] = {} + for version in cfg.game_versions: + out_json["versions"][version] = {} + + out_json["units"] = {} + for version in cfg.game_versions: + out_json["versions"][version]["objdiff"] = str(cfg.get_game_build(version) / "objdiff.json") + + for name, object in objects.items(): + out_json["units"][name] = {} + + for name, object in objects.items(): + out_json["units"][name]["cflags"] = object.options["cflags"] + [f"-DVERSION={version.upper()}"] + out_json["units"][name]["extra_cflags"] = object.options["extra_cflags"] + out_json["units"][name]["mw_version"] = COMPILER_MAP[object.options["mw_version"]] + + cfg.build_path.mkdir(exist_ok=True) + out_path = cfg.build_path / "objdiff_cfg.json" + with out_path.open("w") as f: + json.dump(out_json, f, indent=2) + + +def process_project(cfg: ProjectConfig, args: Any): + objects = cfg.objects() + + create_objdiff_fixup_config(cfg, objects) + + with cfg.build_ninja_path.open("w") as file: + n = ninja_syntax.Writer(file) + + n.rule( + name="download_tool", + command=f'{cfg.python_path} tools/download_tool.py $tool $tag --path $path' + ) + n.newline() + + if cfg.arm7_bios_path.is_file(): + n.variable("arm7_bios_flag", f"--arm7-bios {cfg.arm7_bios_path.relative_to(cfg.root_path)}") + else: + n.variable("arm7_bios_flag", "") + n.newline() + + n.rule( + name="extract", + command=f"{cfg.dsd_path} {cfg.dsd_flags} rom extract --rom $in --output-path $output_path $arm7_bios_flag" + ) + n.newline() + + n.rule( + name="delink", + command=f"{cfg.dsd_path} {cfg.dsd_flags} delink --config-path $config_path" + ) + n.newline() + + n.rule( + name="disassemble", + command=f"{cfg.dsd_path} {cfg.dsd_flags} dis --config-path $config_path --asm-path $output_path --ual" + ) + n.newline() + + # -MMD excludes all includes instead of just system includes for some reason, so use -MD instead. + mwcc_cmd = f'{cfg.wine_path} "{cfg.cc_path}" $cc_flags {cfg.includes} -DVERSION=$game_version -MD -c $in -o $basedir' + mwcc_implicit = [str(cfg.cc_path)] + if cfg.platform.system != "windows": + transform_dep = "tools/transform_dep.py" + mwcc_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" + mwcc_implicit.append(transform_dep) + if cfg.wine_path == cfg.default_wibo_path: + mwcc_implicit.append(cfg.wine_path) + n.rule( + name="mwcc", + command=mwcc_cmd, + depfile="$basefile.d", + ) + n.newline() + + n.rule( + name="lcf", + command=f"{cfg.dsd_path} {cfg.dsd_flags} lcf -c $config_path" + ) + n.newline() + + n.rule( + name="mwld", + command=f'{cfg.wine_path} "{cfg.ld_path}" {' '.join(cfg.ldflags)} $extra_ld_flags @$objects_file $lcf_file -o $out' + ) + n.newline() + + n.rule( + name="rom_config", + command=f"{cfg.dsd_path} {cfg.dsd_flags} rom config --elf $in --config $config_path" + ) + n.newline() + + n.rule( + name="rom_build", + command=f"{cfg.dsd_path} {cfg.dsd_flags} rom build --config $in --rom $out $arm7_bios_flag" + ) + n.newline() + + cflags = " ".join(cfg.cflags_base) + dsd_objdiff_args = " ".join([ + "--scratch", # Metadata for creating decomp.me scratches + f"--compiler {cfg.get_decompme_compiler()}", # decomp.me compiler name + f'--c-flags "{cflags} -lang=c++"', # decomp.me compiler flags + "--custom-make ninja", # Command for rebuilding files + ]) + n.rule( + name="objdiff", + command=f"touch {cfg.dsd_path} && {cfg.dsd_path} {cfg.dsd_flags} objdiff --config-path $config_path --output-path $out_path {dsd_objdiff_args}" + ) + n.newline() + + n.rule( + name="objdiff_report", + command=f"{cfg.objdiff_path} -C $dir report generate -o $filename" + ) + n.newline() + + n.rule( + name="m2ctx", + command=f"{cfg.python_path} tools/m2ctx.py -g $version -f $out $in" + ) + n.newline() + + n.rule( + name="check_modules", + command=f"{cfg.dsd_path} {cfg.dsd_flags} check modules --config-path $config_path --fail" + ) + n.newline() + + n.rule( + name="check_symbols", + command=f"{cfg.dsd_path} {cfg.dsd_flags} check symbols --config-path $config_path --elf-path $elf_path --fail --max-lines 20" + ) + n.newline() + + n.rule( + name="apply", + command=f"{cfg.dsd_path} {cfg.dsd_flags} apply --config-path $config_path --elf-path $elf_path" + ) + n.newline() + + n.rule( + name="sha1", + command=f"{cfg.python_path} tools/sha1.py $in -c $sha1_file" + ) + n.newline() + + configure_cmdline = subprocess.list2cmdline(sys.argv[1:]) + n.rule( + name="configure", + command=f"{cfg.python_path} tools/configure.py {configure_cmdline}", + generator=True + ) + n.newline() + + n.rule( + name="format_delinks", + command=f"{cfg.python_path} tools/format_delinks.py" + ) + n.newline() + + n.rule( + name="post_objdiff", + command=f"{cfg.python_path} tools/objdiff_config.py" + ) + n.newline() + + add_download_tool_builds(cfg, n, args) + add_configure_build(cfg, n) + + if cfg.check_can_run_dsd(): + defaults = [] + + for version in cfg.game_versions: + add_extract_build(cfg, version, n, args) + add_delink_and_lcf_builds(cfg, version, n) + add_disassemble_builds(cfg, version, n) + add_mwcc_builds(cfg, version, objects, n, mwcc_implicit) + add_mwld_and_rom_builds(cfg, version, n) + add_check_builds(cfg, version, n) + add_objdiff_builds(cfg, version, n) + add_apply_build(cfg, version, n) + defaults.extend([f"check_{version}", f"sha1_{version}"]) + + n.build( + rule="post_objdiff", + implicit=[f"objdiff_{version}.json" for version in cfg.game_versions], + outputs="objdiff" + ) + n.newline() + + n.build( + rule="format_delinks", + outputs="format_delinks" + ) + n.newline() + + n.default(["format_delinks", "objdiff", *defaults]) + else: + n.default(["download_tools"])