From db2b3f79c74fe4126f919b88b574ad04cb3dd8c2 Mon Sep 17 00:00:00 2001 From: Aetias Date: Sun, 15 Sep 2024 22:29:02 +0200 Subject: [PATCH] Initial Ninja build script --- .gitignore | 2 + tools/.gitignore | 1 + tools/configure.py | 229 +++++++++++++++++++++++++++++++++++++++++ tools/ninja_syntax.py | 231 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 463 insertions(+) create mode 100644 tools/configure.py create mode 100644 tools/ninja_syntax.py diff --git a/.gitignore b/.gitignore index edae797c..e2b78543 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ objdiff.json /dsd /dsd.exe /dsd.pdb +build.ninja +.ninja_log diff --git a/tools/.gitignore b/tools/.gitignore index a013c9f3..4d171cab 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -1,3 +1,4 @@ mwccarm/ temp/ deps/ +__pycache__/ diff --git a/tools/configure.py b/tools/configure.py new file mode 100644 index 00000000..e396cc30 --- /dev/null +++ b/tools/configure.py @@ -0,0 +1,229 @@ +#!/usr/bin/python3 + +import os +from pathlib import Path +import platform + +import ninja_syntax + + +# Config +MWCC_VERSION = "2.0/sp1p5" +CC_FLAGS = " ".join([ + "-O4,p", # Optimize maximally for performance + "-enum int", # Use int-sized enums + "-char signed", # Char type is signed + "-str noreuse", # Equivalent strings are different objects + "-proc arm946e", # Target processor + "-gccext,on", # Enable GCC extensions + "-fp soft", # Compute float operations in software + "-inline on,noauto", # Inline only functions marked with 'inline' + "-Cpp_exceptions off", # Disable C++ exceptions + "-RTTI off", # Disable runtime type information + "-interworking", # Enable ARM/Thumb interworking + "-sym on", # Debug info, including line numbers + "-gccinc", # Interpret #include "..." and #include <...> equally + "-nolink", # Do not link + "-msgstyle gcc", # Use GCC-like messages (some IDEs will make file names clickable) +]) +LD_FLAGS = " ".join([ + "-proc arm946e", # Target processor + "-nostdlib", # No C/C++ standard library + "-interworking", # Enable ARM/Thumb interworking + "-nodead", # No dead code elimination + "-m Entry", # Set entry function + "-map closure,unused", # Generate map file + "-msgstyle gcc", # Use GCC-like messages (some IDEs will make file names clickable) +]) + + +# Paths +current_path = Path(__name__) +root_path = current_path.parent +build_ninja_path = root_path / "build.ninja" +arm7_bios_path = root_path / "arm7_bios.bin" +config_path = root_path / "config" +build_path = root_path / "build" +src_path = root_path / "src" +libs_path = root_path / "libs" +tools_path = root_path / "tools" +mwcc_path = tools_path / "mwccarm" / MWCC_VERSION +mw_license_path = tools_path / "mwccarm" / "license.dat" + + +# Includes +includes = [ + str(root_path / "include") +] +for root, dirs, _ in os.walk(libs_path): + for dir in dirs: + if dir == "include": + includes.append(Path(root) / dir) +CC_INCLUDES = " ".join(f"-i {include}" for include in includes) + + +# Platform info +EXE = "" +WINE = "" +SHELL = "" +MW_LICENSE_SHELL = "" +system = platform.system() +if system == "Windows" or system.startswith("MSYS_NT"): + system = "windows" + EXE = ".exe" + SHELL = "cmd /c" + MW_LICENSE_SHELL = f"{SHELL} set LM_LICENSE_FILE={mw_license_path} &&" +elif system == "Linux": + system = "linux" + WINE = "wine" + MW_LICENSE_SHELL = f"LM_LICENSE_FILE={mw_license_path}" +else: + print(f"Unknown system '{system}'") + exit(1) +match platform.machine().lower(): + case "amd64" | "x86_64": machine = "x86_64" + case machine: + print(f"Unknown machine: {machine}") + exit(1) + + +def main(): + with build_ninja_path.open("w") as file: + n = ninja_syntax.Writer(file) + + if arm7_bios_path.is_file(): + n.variable("arm7_bios_flag", f"--arm7-bios {arm7_bios_path.relative_to(root_path)}") + else: + n.variable("arm7_bios_flag", "") + n.newline() + + n.rule( + name="delink", + command="./dsd delink --config-path $config_path --elf-path $delinks_path" + ) + n.newline() + + n.rule( + name="mwcc", + command=f'{MW_LICENSE_SHELL} {WINE} "{mwcc_path}/mwccarm.exe" {CC_FLAGS} {CC_INCLUDES} $cc_flags -d $game_code $in -o $out' + ) + n.newline() + + n.rule( + name="lcf", + command="./dsd lcf -c $config_path --lcf-file $lcf_file --objects-file $objects_file --objects-path $objects_path --build-path $build_path" + ) + n.newline() + + n.rule( + name="mwld", + command=f'{MW_LICENSE_SHELL} {WINE} "{mwcc_path}/mwldarm.exe" {LD_FLAGS} @$objects_file $lcf_file -o $out' + ) + n.newline() + + for game_code in os.listdir(config_path): + game_config = config_path / game_code + if not game_config.is_dir(): continue + game_build = build_path / game_code + + n.comment(game_code) + add_delink_and_lcf_builds(n, game_config, game_build) + add_mwcc_builds(n, game_code, game_build) + + source_object_files = [ + str(game_build / source_file) + ".o" + for source_file in get_c_cpp_files([src_path, libs_path]) + ] + lcf_file = str(game_build / "linker_script.lcf") + objects_file = str(game_build / "objects.txt") + output_file = game_build / "arm9.o" + n.build( + inputs=source_object_files + [lcf_file, objects_file], + rule="mwld", + outputs=str(output_file), + variables={ + "target_dir": game_build, + "objects_file": objects_file, + "lcf_file": lcf_file, + } + ) + + +def add_mwcc_builds(n: ninja_syntax.Writer, game_code: str, game_build: Path): + for source_file in get_c_cpp_files([src_path, libs_path]): + output_file = str(game_build / source_file) + ".o" + cc_flags = [] + if is_cpp(source_file): cc_flags.append("-lang=c++") + elif is_c(source_file): cc_flags.append("-lang=c") + n.build( + inputs=str(source_file), + rule="mwcc", + outputs=output_file, + variables={ + "game_code": game_code, + "cc_flags": " ".join(cc_flags) + } + ) + n.newline() + + +def get_c_cpp_files(dirs: list[Path]): + for dir in dirs: + for root, _, files in os.walk(dir): + root = Path(root) + for file in files: + if is_cpp(file) or is_c(file): + yield root / file + + +def is_cpp(name: str): + return Path(name).suffix in [".cpp"] + + +def is_c(name: str): + return Path(name).suffix in [".c"] + + +def add_delink_and_lcf_builds(n: ninja_syntax.Writer, game_config: Path, game_build: Path): + n.comment("Delink ELF binaries when any delinks.txt file is modified") + delinks_files = get_config_files(game_config, "delinks.txt") + relocs_files = get_config_files(game_config, "relocs.txt") + symbols_files = get_config_files(game_config, "symbols.txt") + delinks_path = game_build / "delinks" + n.build( + inputs=delinks_files + relocs_files + symbols_files, + rule="delink", + outputs=str(delinks_path / "delink.yaml"), + variables={ + "config_path": game_config / "arm9" / "config.yaml", + "delinks_path": delinks_path, + } + ) + n.newline() + lcf_file = game_build / "linker_script.lcf" + objects_file = game_build / "objects.txt" + n.build( + inputs=delinks_files, + rule="lcf", + outputs=[str(lcf_file), str(objects_file)], + variables={ + "config_path": game_config / "arm9" / "config.yaml", + "lcf_file": lcf_file, + "objects_file": objects_file, + "objects_path": delinks_path, + "build_path": game_build, + } + ) + n.newline() + + +def get_config_files(game_config: Path, name: str): + return [ + f"{root}/{file}" + for root, _, files in os.walk(game_config) + for file in files + if file == name + ] + + +if __name__ == "__main__": main() diff --git a/tools/ninja_syntax.py b/tools/ninja_syntax.py new file mode 100644 index 00000000..2aa8456e --- /dev/null +++ b/tools/ninja_syntax.py @@ -0,0 +1,231 @@ +#!/usr/bin/python + +# Copyright 2011 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Python module for generating .ninja files. + +Note that this is emphatically not a required piece of Ninja; it's +just a helpful utility for build-file-generation systems that already +use Python. +""" + +import re +import textwrap +from io import TextIOWrapper +from typing import Dict, List, Match, Optional, Tuple, Union + +def escape_path(word: str) -> str: + return word.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:') + +class Writer(object): + def __init__(self, output: TextIOWrapper, width: int = 78) -> None: + self.output = output + self.width = width + + def newline(self) -> None: + self.output.write('\n') + + def comment(self, text: str) -> None: + for line in textwrap.wrap(text, self.width - 2, break_long_words=False, + break_on_hyphens=False): + self.output.write('# ' + line + '\n') + + def variable( + self, + key: str, + value: Optional[Union[bool, int, float, str, List[str]]], + indent: int = 0, + ) -> None: + if value is None: + return + if isinstance(value, list): + value = ' '.join(filter(None, value)) # Filter out empty strings. + self._line('%s = %s' % (key, value), indent) + + def pool(self, name: str, depth: int) -> None: + self._line('pool %s' % name) + self.variable('depth', depth, indent=1) + + def rule( + self, + name: str, + command: str, + description: Optional[str] = None, + depfile: Optional[str] = None, + generator: bool = False, + pool: Optional[str] = None, + restat: bool = False, + rspfile: Optional[str] = None, + rspfile_content: Optional[str] = None, + deps: Optional[Union[str, List[str]]] = None, + ) -> None: + self._line('rule %s' % name) + self.variable('command', command, indent=1) + if description: + self.variable('description', description, indent=1) + if depfile: + self.variable('depfile', depfile, indent=1) + if generator: + self.variable('generator', '1', indent=1) + if pool: + self.variable('pool', pool, indent=1) + if restat: + self.variable('restat', '1', indent=1) + if rspfile: + self.variable('rspfile', rspfile, indent=1) + if rspfile_content: + self.variable('rspfile_content', rspfile_content, indent=1) + if deps: + self.variable('deps', deps, indent=1) + + def build( + self, + outputs: Union[str, List[str]], + rule: str, + inputs: Optional[Union[str, List[str]]] = None, + implicit: Optional[Union[str, List[str]]] = None, + order_only: Optional[Union[str, List[str]]] = None, + variables: Optional[ + Union[ + List[Tuple[str, Optional[Union[str, List[str]]]]], + Dict[str, Optional[Union[str, List[str]]]], + ] + ] = None, + implicit_outputs: Optional[Union[str, List[str]]] = None, + pool: Optional[str] = None, + dyndep: Optional[str] = None, + ) -> List[str]: + outputs = as_list(outputs) + out_outputs = [escape_path(x) for x in outputs] + all_inputs = [escape_path(x) for x in as_list(inputs)] + + if implicit: + implicit = [escape_path(x) for x in as_list(implicit)] + all_inputs.append('|') + all_inputs.extend(implicit) + if order_only: + order_only = [escape_path(x) for x in as_list(order_only)] + all_inputs.append('||') + all_inputs.extend(order_only) + if implicit_outputs: + implicit_outputs = [escape_path(x) + for x in as_list(implicit_outputs)] + out_outputs.append('|') + out_outputs.extend(implicit_outputs) + + self._line('build %s: %s' % (' '.join(out_outputs), + ' '.join([rule] + all_inputs))) + if pool is not None: + self._line(' pool = %s' % pool) + if dyndep is not None: + self._line(' dyndep = %s' % dyndep) + + if variables: + if isinstance(variables, dict): + iterator = iter(variables.items()) + else: + iterator = iter(variables) + + for key, val in iterator: + self.variable(key, val, indent=1) + + return outputs + + def include(self, path: str) -> None: + self._line('include %s' % path) + + def subninja(self, path: str) -> None: + self._line('subninja %s' % path) + + def default(self, paths: Union[str, List[str]]) -> None: + self._line('default %s' % ' '.join(as_list(paths))) + + def _count_dollars_before_index(self, s: str, i: int) -> int: + """Returns the number of '$' characters right in front of s[i].""" + dollar_count = 0 + dollar_index = i - 1 + while dollar_index > 0 and s[dollar_index] == '$': + dollar_count += 1 + dollar_index -= 1 + return dollar_count + + def _line(self, text: str, indent: int = 0) -> None: + """Write 'text' word-wrapped at self.width characters.""" + leading_space = ' ' * indent + while len(leading_space) + len(text) > self.width: + # The text is too wide; wrap if possible. + + # Find the rightmost space that would obey our width constraint and + # that's not an escaped space. + available_space = self.width - len(leading_space) - len(' $') + space = available_space + while True: + space = text.rfind(' ', 0, space) + if (space < 0 or + self._count_dollars_before_index(text, space) % 2 == 0): + break + + if space < 0: + # No such space; just use the first unescaped space we can find. + space = available_space - 1 + while True: + space = text.find(' ', space + 1) + if (space < 0 or + self._count_dollars_before_index(text, space) % 2 == 0): + break + if space < 0: + # Give up on breaking. + break + + self.output.write(leading_space + text[0:space] + ' $\n') + text = text[space+1:] + + # Subsequent lines are continuations, so indent them. + leading_space = ' ' * (indent+2) + + self.output.write(leading_space + text + '\n') + + def close(self) -> None: + self.output.close() + + +def as_list(input: Optional[Union[str, List[str]]]) -> List[str]: + if input is None: + return [] + if isinstance(input, list): + return input + return [input] + + +def escape(string: str) -> str: + """Escape a string such that it can be embedded into a Ninja file without + further interpretation.""" + assert '\n' not in string, 'Ninja syntax does not allow newlines' + # We only have one special metacharacter: '$'. + return string.replace('$', '$$') + + +def expand(string: str, vars: Dict[str, str], local_vars: Dict[str, str] = {}) -> str: + """Expand a string containing $vars as Ninja would. + + Note: doesn't handle the full Ninja variable syntax, but it's enough + to make configure.py's use of it work. + """ + def exp(m: Match[str]) -> str: + var = m.group(1) + if var == '$': + return '$' + return local_vars.get(var, vars.get(var, '')) + return re.sub(r'\$(\$|\w*)', exp, string)