From 1fc8c3a9bbbbc70fb8f9aea3a94176cac01ce21c Mon Sep 17 00:00:00 2001 From: Yanis42 <35189056+Yanis42@users.noreply.github.com> Date: Mon, 16 Sep 2024 14:00:37 +0200 Subject: [PATCH] initial commit --- .gitattributes | 13 +++ .gitignore | 13 +++ LICENSE | 121 ++++++++++++++++++++ README | 83 ++++++++++++++ config/BKIJ01/config.yaml | 0 orig/BKIJ01/.gitkeep | 0 tools/configure.py | 231 ++++++++++++++++++++++++++++++++++++++ tools/m2ctx.py | 87 ++++++++++++++ tools/mangle.py | 70 ++++++++++++ tools/ninja_syntax.py | 231 ++++++++++++++++++++++++++++++++++++++ tools/requirements.txt | 2 + tools/setup.py | 50 +++++++++ 12 files changed, 901 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README create mode 100644 config/BKIJ01/config.yaml create mode 100644 orig/BKIJ01/.gitkeep create mode 100644 tools/configure.py create mode 100644 tools/m2ctx.py create mode 100644 tools/mangle.py create mode 100644 tools/ninja_syntax.py create mode 100644 tools/requirements.txt create mode 100644 tools/setup.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..47d43017 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,13 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# Explicitly declare text files +*.py text + +# Enforce platform-specific encodings +*.bat text eol=crlf +*.sh text eol=lf +*.sha1 text eol=lf + +# decomp-toolkit writes files with LF +config/**/*.txt text eol=lf diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..47e8153c --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +__pycache__ +.idea +.vscode +.ninja_* +.mypy_cache +*.exe +build +build.ninja +objdiff.json +orig/*/* +!orig/*/.gitkeep +/*.txt +ctx.c diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..0e259d42 --- /dev/null +++ b/LICENSE @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/README b/README new file mode 100644 index 00000000..4406847a --- /dev/null +++ b/README @@ -0,0 +1,83 @@ +# The Legend of Zelda: Spirit Tracks + +A work-in-progress decompilation of The Legend of Zelda: Spirit Tracks. + +Supported versions: +- `st-j`: Spirit Tracks - Japan + +Currently the decompilation mainly targets the `st-j` version. + +## Building + +### Requirements + +You will need the following dependencies: +* git +* ninja +* python3 +* wine (for macOS or non-x86 Linux) +* clang-format (optional) + +#### Ubuntu/Debian/Windows (WSL) + +You can install the dependencies with the following commands: + +``` +sudo apt-get update +sudo apt-get install git ninja-build python3 +``` + +#### macOS + +You can install dependencies via Homebrew with the following command: + +``` +brew install git ninja python3 +brew install --cask --no-quarantine gcenx/wine/wine-crossover +``` + +#### Windows (Native) + +You will need the following dependencies: +- [ninja.exe](https://github.com/ninja-build/ninja/releases/latest) +- Python (make sure to add it to your PATH during the installation) +- [Git for Windows](https://www.git-scm.com/downloads) + +You need to add ``C:\Program Files\Git\bin`` to your system's PATH (not the user one) in order to execute bash scripts properly. + +To get objdiff to work properly you also need to add the path to the folder containing ``ninja.exe`` to the system's PATH. + +### Instructions + +1. Clone the repo using `git clone https://github.com/yanis42/st`. + +2. Extract the following TGC archive containing the N64 emulator from the disc of the version you want to build: + + * `mq-j`: `zlj_f.tgc` + * `mq-u`: `zlj_f.tgc` + * `mq-e`: `zlj_f.tgc` + * `ce-j`: `120903_zelda.tgc` + * `ce-u`: `zelda_ENG_090903.tgc` + * `ce-e`: `zelda_PAL_093003.tgc` + + Then, extract the DOL file from the TGC archive and place it in the repo as `orig//main.dol`. + + You can use [Dolphin](https://dolphin-emu.org) to perform both of these extraction steps: + first, right click on the `.iso` file, select "Properties", go to the "Filesystem" tab, find the correct + `.tgc` file, then right-click and select "Extract File..." and extract it to your games folder. + Then, right-click the extracted `.tgc` file in Dolphin, select "Properties", go to the "Filesystem" tab, + right-click the "Disc" and select "Extract System Data..." to extract the DOL file. + +3. Run `python3 tools/configure.py` to generate the build. (Note: on Windows you might need to run ``python tools/configure.py``.) + +4. Run `ninja` to build the `st-j` version, or run `ninja ` to build another version. + +## Development Tools + +### objdiff + +For local decompilation testing, start the [objdiff](https://github.com/encounter/objdiff) GUI and open this directory as the project directory. + +## License + +This project is licensed under CC0. Some tools might have their own license. diff --git a/config/BKIJ01/config.yaml b/config/BKIJ01/config.yaml new file mode 100644 index 00000000..e69de29b diff --git a/orig/BKIJ01/.gitkeep b/orig/BKIJ01/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tools/configure.py b/tools/configure.py new file mode 100644 index 00000000..f227f40d --- /dev/null +++ b/tools/configure.py @@ -0,0 +1,231 @@ +#!/usr/bin/python3 + +import os +from pathlib import Path +import platform + +import ninja_syntax + + +# Config +MWCC_VERSION = "2.0/sp1p5" +CC_FLAGS = " ".join([ + "-O4,p", # Optimize maximally for performance + "-enum int", # Use int-sized enums + "-char signed", # Char type is signed + "-str noreuse", # Equivalent strings are different objects + "-proc arm946e", # Target processor + "-gccext,on", # Enable GCC extensions + "-fp soft", # Compute float operations in software + "-inline on,noauto", # Inline only functions marked with 'inline' + "-Cpp_exceptions off", # Disable C++ exceptions + "-RTTI off", # Disable runtime type information + "-interworking", # Enable ARM/Thumb interworking + "-sym on", # Debug info, including line numbers + "-gccinc", # Interpret #include "..." and #include <...> equally + "-nolink", # Do not link + "-msgstyle gcc", # Use GCC-like messages (some IDEs will make file names clickable) +]) +LD_FLAGS = " ".join([ + "-proc arm946e", # Target processor + "-nostdlib", # No C/C++ standard library + "-interworking", # Enable ARM/Thumb interworking + "-nodead", # No dead code elimination + "-m Entry", # Set entry function + "-map closure,unused", # Generate map file + "-msgstyle gcc", # Use GCC-like messages (some IDEs will make file names clickable) +]) + + +# Paths +current_path = Path(__name__) +root_path = current_path.parent +build_ninja_path = root_path / "build.ninja" +arm7_bios_path = root_path / "arm7_bios.bin" +config_path = root_path / "config" +build_path = root_path / "build" +src_path = root_path / "src" +libs_path = root_path / "libs" +tools_path = root_path / "tools" +mwcc_path = tools_path / "mwccarm" / MWCC_VERSION +mw_license_path = tools_path / "mwccarm" / "license.dat" + + +# Includes +includes = [ + str(root_path / "include") +] +for root, dirs, _ in os.walk(libs_path): + for dir in dirs: + if dir == "include": + includes.append(Path(root) / dir) +CC_INCLUDES = " ".join(f"-i {include}" for include in includes) + + +# Platform info +EXE = "" +WINE = "" +SHELL = "" +MW_LICENSE_SHELL = "" +system = platform.system() +if system == "Windows" or system.startswith("MSYS_NT"): + system = "windows" + EXE = ".exe" + SHELL = "cmd /c" + MW_LICENSE_SHELL = f"{SHELL} set LM_LICENSE_FILE={mw_license_path} &&" + DSD = "dsd.exe" +elif system == "Linux": + system = "linux" + WINE = "wine" + MW_LICENSE_SHELL = f"LM_LICENSE_FILE={mw_license_path}" + DSD = "dsd" +else: + print(f"Unknown system '{system}'") + exit(1) +match platform.machine().lower(): + case "amd64" | "x86_64": machine = "x86_64" + case machine: + print(f"Unknown machine: {machine}") + exit(1) + + +def main(): + with build_ninja_path.open("w") as file: + n = ninja_syntax.Writer(file) + + if arm7_bios_path.is_file(): + n.variable("arm7_bios_flag", f"--arm7-bios {arm7_bios_path.relative_to(root_path)}") + else: + n.variable("arm7_bios_flag", "") + n.newline() + + n.rule( + name="delink", + command=f"./{DSD} delink --config-path $config_path --elf-path $delinks_path" + ) + n.newline() + + n.rule( + name="mwcc", + command=f'{MW_LICENSE_SHELL} {WINE} "{mwcc_path}/mwccarm.exe" {CC_FLAGS} {CC_INCLUDES} $cc_flags -d $game_code $in -o $out' + ) + n.newline() + + n.rule( + name="lcf", + command=f"./{DSD} lcf -c $config_path --lcf-file $lcf_file --objects-file $objects_file --objects-path $objects_path --build-path $build_path" + ) + n.newline() + + n.rule( + name="mwld", + command=f'{MW_LICENSE_SHELL} {WINE} "{mwcc_path}/mwldarm.exe" {LD_FLAGS} @$objects_file $lcf_file -o $out' + ) + n.newline() + + for game_code in os.listdir(config_path): + game_config = config_path / game_code + if not game_config.is_dir(): continue + game_build = build_path / game_code + + n.comment(game_code) + add_delink_and_lcf_builds(n, game_config, game_build) + add_mwcc_builds(n, game_code, game_build) + + source_object_files = [ + str(game_build / source_file) + ".o" + for source_file in get_c_cpp_files([src_path, libs_path]) + ] + lcf_file = str(game_build / "linker_script.lcf") + objects_file = str(game_build / "objects.txt") + output_file = game_build / "arm9.o" + n.build( + inputs=source_object_files + [lcf_file, objects_file], + rule="mwld", + outputs=str(output_file), + variables={ + "target_dir": game_build, + "objects_file": objects_file, + "lcf_file": lcf_file, + } + ) + + +def add_mwcc_builds(n: ninja_syntax.Writer, game_code: str, game_build: Path): + for source_file in get_c_cpp_files([src_path, libs_path]): + output_file = str(game_build / source_file) + ".o" + cc_flags = [] + if is_cpp(source_file): cc_flags.append("-lang=c++") + elif is_c(source_file): cc_flags.append("-lang=c") + n.build( + inputs=str(source_file), + rule="mwcc", + outputs=output_file, + variables={ + "game_code": game_code, + "cc_flags": " ".join(cc_flags) + } + ) + n.newline() + + +def get_c_cpp_files(dirs: list[Path]): + for dir in dirs: + for root, _, files in os.walk(dir): + root = Path(root) + for file in files: + if is_cpp(file) or is_c(file): + yield root / file + + +def is_cpp(name: str): + return Path(name).suffix in [".cpp"] + + +def is_c(name: str): + return Path(name).suffix in [".c"] + + +def add_delink_and_lcf_builds(n: ninja_syntax.Writer, game_config: Path, game_build: Path): + n.comment("Delink ELF binaries when any delinks.txt file is modified") + delinks_files = get_config_files(game_config, "delinks.txt") + relocs_files = get_config_files(game_config, "relocs.txt") + symbols_files = get_config_files(game_config, "symbols.txt") + delinks_path = game_build / "delinks" + n.build( + inputs=delinks_files + relocs_files + symbols_files, + rule="delink", + outputs=str(delinks_path / "delink.yaml"), + variables={ + "config_path": game_config / "arm9" / "config.yaml", + "delinks_path": delinks_path, + } + ) + n.newline() + lcf_file = game_build / "linker_script.lcf" + objects_file = game_build / "objects.txt" + n.build( + inputs=delinks_files, + rule="lcf", + outputs=[str(lcf_file), str(objects_file)], + variables={ + "config_path": game_config / "arm9" / "config.yaml", + "lcf_file": lcf_file, + "objects_file": objects_file, + "objects_path": delinks_path, + "build_path": game_build, + } + ) + n.newline() + + +def get_config_files(game_config: Path, name: str): + return [ + f"{root}/{file}" + for root, _, files in os.walk(game_config) + for file in files + if file == name + ] + + +if __name__ == "__main__": main() diff --git a/tools/m2ctx.py b/tools/m2ctx.py new file mode 100644 index 00000000..dc43f326 --- /dev/null +++ b/tools/m2ctx.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python + +import sys +import pyperclip +import subprocess +import os +from pathlib import Path +import argparse +import re +import tempfile + +parser = argparse.ArgumentParser(description="Generates a context for decomp.me") +parser.add_argument('file', help="Input file to preprocess") +parser.add_argument('-f', type=str, dest='out_file', required=False, help='Output context file') +parser.add_argument('-c', action=argparse.BooleanOptionalAction, dest='clipboard', required=False, help='Copy output to clipboard') +parser.add_argument('-e', type=str, dest='encoding', required=False, default="utf-8", help='Input file encoding') +parser.add_argument('-v', action=argparse.BooleanOptionalAction, dest='verbose', required=False, help='Verbose error output') +args = parser.parse_args() + +CXX_FLAGS = [ + '-nostdinc', + '-Iinclude', + '-Ilibs/c/include', + '-Ilibs/cpp/include' +] + +script_dir = Path(os.path.dirname(os.path.realpath(__file__))) +root_dir = script_dir / ".." + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +# Finds all lines starting with #include followed by <...> or "..." +INCLUDE_REGEX = r'^\s*#\s*include\s*([<"][\S ]+[>"])\s*$' +# Finds all line comments and multiline comments +COMMENT_REGEX = r'\/\/.*$|\/\*(?:.|\r|\n)+?\*\/' + +with open(args.file, 'r') as f: + contents = f.read() +contents = re.sub(COMMENT_REGEX, '', contents, 0, re.MULTILINE) +includes = re.findall(INCLUDE_REGEX, contents, re.MULTILINE) + +_, suffix = os.path.splitext(args.file) + +with tempfile.NamedTemporaryFile(delete=True, suffix=suffix) as tmp_file: + # Write includes + for include in includes: + tmp_file.write(f'#include {include}\n'.encode()) + tmp_file.flush() + + # Run preprocessor + try: + ctx: str = subprocess.check_output([ + 'gcc', + '-E', '-P', '-fworking-directory', '-undef', '-dD', + *CXX_FLAGS, + tmp_file.name + ], cwd=root_dir, encoding=args.encoding) + except subprocess.CalledProcessError as e: + eprint(f"Failed to preprocess '{args.file}'") + if args.verbose: eprint(e) + else: eprint("Use -v for more verbose error output") + exit(1) + +lines = ctx.splitlines(True) +for i in reversed(range(len(lines))): + if lines[i].startswith('#define __cplusplus'): lines.pop(i) + elif lines[i].startswith('#define __STDC_HOSTED__'): lines.pop(i) + elif lines[i].startswith('#define __STDC__'): lines.pop(i) + elif lines[i].startswith('#define __STDC_VERSION__'): lines.pop(i) + +ctx = ''.join(lines) + +if args.out_file: + try: + with open(args.out_file, "w") as file: + file.write(ctx) + except OSError as e: + eprint(f"Failed to write file '{args.out_file}'") + if args.verbose: eprint(e) + else: eprint("Use -v for more verbose error output") + exit(1) +if args.clipboard: + pyperclip.copy(ctx) + eprint("Copied context to clipboard") +if args.out_file is None and not args.clipboard: + print(ctx) diff --git a/tools/mangle.py b/tools/mangle.py new file mode 100644 index 00000000..d14bec54 --- /dev/null +++ b/tools/mangle.py @@ -0,0 +1,70 @@ +import argparse +import os +from pathlib import Path +import platform +import re +import subprocess + +parser = argparse.ArgumentParser(description='Mangles function declarations') +parser.add_argument('file', help='C/C++ source file with defined function to mangle') + +tools_dir = Path(os.path.dirname(os.path.realpath(__file__))) +cc_path = tools_dir / 'mwccarm' / '2.0' / 'sp1p5' / 'mwccarm.exe' +root_dir = tools_dir.parent +include_dir = root_dir / 'include' +libs_dir = root_dir / 'libs' +libc_include_dir = libs_dir / 'c' / 'include' +libcpp_include_dir = libs_dir / 'cpp' / 'include' + +if platform.system() == 'Windows': cc = [str(cc_path)] +else: cc = ['wine', str(cc_path)] + +args = parser.parse_args() + +cc.extend([ + '-enum', 'int', + '-char', 'signed', + '-proc', 'arm946e', + '-gccext,on', + '-fp', 'soft', + '-inline', 'on,noauto', + '-Cpp_exceptions', 'off', + '-RTTI', 'off', + '-interworking', + '-nolink', + '-msgstyle', 'gcc', + '-dis', + '-gccinc', + '-i', include_dir, + '-i', libc_include_dir, + '-i', libcpp_include_dir, + args.file +]) + +try: + output = subprocess.check_output(cc) +except subprocess.CalledProcessError as e: + print(e.stdout.decode()) + exit(1) + +output = output.decode() + +# print(output) + +mangled_funcs: list[str] = re.findall(r'.text +([^\$ ]\S+)', output) +mangled_data: list[str] = re.findall(r'(?:.data|.bss) +([^\. ]\S+)', output) + +if len(mangled_funcs) > 0: + print('Functions:') + print() + for func in mangled_funcs: + print(func) + print() + print() +if len(mangled_data) > 0: + print('Data:') + print() + for data in mangled_data: + print(data) + print() + print() diff --git a/tools/ninja_syntax.py b/tools/ninja_syntax.py new file mode 100644 index 00000000..2aa8456e --- /dev/null +++ b/tools/ninja_syntax.py @@ -0,0 +1,231 @@ +#!/usr/bin/python + +# Copyright 2011 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Python module for generating .ninja files. + +Note that this is emphatically not a required piece of Ninja; it's +just a helpful utility for build-file-generation systems that already +use Python. +""" + +import re +import textwrap +from io import TextIOWrapper +from typing import Dict, List, Match, Optional, Tuple, Union + +def escape_path(word: str) -> str: + return word.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:') + +class Writer(object): + def __init__(self, output: TextIOWrapper, width: int = 78) -> None: + self.output = output + self.width = width + + def newline(self) -> None: + self.output.write('\n') + + def comment(self, text: str) -> None: + for line in textwrap.wrap(text, self.width - 2, break_long_words=False, + break_on_hyphens=False): + self.output.write('# ' + line + '\n') + + def variable( + self, + key: str, + value: Optional[Union[bool, int, float, str, List[str]]], + indent: int = 0, + ) -> None: + if value is None: + return + if isinstance(value, list): + value = ' '.join(filter(None, value)) # Filter out empty strings. + self._line('%s = %s' % (key, value), indent) + + def pool(self, name: str, depth: int) -> None: + self._line('pool %s' % name) + self.variable('depth', depth, indent=1) + + def rule( + self, + name: str, + command: str, + description: Optional[str] = None, + depfile: Optional[str] = None, + generator: bool = False, + pool: Optional[str] = None, + restat: bool = False, + rspfile: Optional[str] = None, + rspfile_content: Optional[str] = None, + deps: Optional[Union[str, List[str]]] = None, + ) -> None: + self._line('rule %s' % name) + self.variable('command', command, indent=1) + if description: + self.variable('description', description, indent=1) + if depfile: + self.variable('depfile', depfile, indent=1) + if generator: + self.variable('generator', '1', indent=1) + if pool: + self.variable('pool', pool, indent=1) + if restat: + self.variable('restat', '1', indent=1) + if rspfile: + self.variable('rspfile', rspfile, indent=1) + if rspfile_content: + self.variable('rspfile_content', rspfile_content, indent=1) + if deps: + self.variable('deps', deps, indent=1) + + def build( + self, + outputs: Union[str, List[str]], + rule: str, + inputs: Optional[Union[str, List[str]]] = None, + implicit: Optional[Union[str, List[str]]] = None, + order_only: Optional[Union[str, List[str]]] = None, + variables: Optional[ + Union[ + List[Tuple[str, Optional[Union[str, List[str]]]]], + Dict[str, Optional[Union[str, List[str]]]], + ] + ] = None, + implicit_outputs: Optional[Union[str, List[str]]] = None, + pool: Optional[str] = None, + dyndep: Optional[str] = None, + ) -> List[str]: + outputs = as_list(outputs) + out_outputs = [escape_path(x) for x in outputs] + all_inputs = [escape_path(x) for x in as_list(inputs)] + + if implicit: + implicit = [escape_path(x) for x in as_list(implicit)] + all_inputs.append('|') + all_inputs.extend(implicit) + if order_only: + order_only = [escape_path(x) for x in as_list(order_only)] + all_inputs.append('||') + all_inputs.extend(order_only) + if implicit_outputs: + implicit_outputs = [escape_path(x) + for x in as_list(implicit_outputs)] + out_outputs.append('|') + out_outputs.extend(implicit_outputs) + + self._line('build %s: %s' % (' '.join(out_outputs), + ' '.join([rule] + all_inputs))) + if pool is not None: + self._line(' pool = %s' % pool) + if dyndep is not None: + self._line(' dyndep = %s' % dyndep) + + if variables: + if isinstance(variables, dict): + iterator = iter(variables.items()) + else: + iterator = iter(variables) + + for key, val in iterator: + self.variable(key, val, indent=1) + + return outputs + + def include(self, path: str) -> None: + self._line('include %s' % path) + + def subninja(self, path: str) -> None: + self._line('subninja %s' % path) + + def default(self, paths: Union[str, List[str]]) -> None: + self._line('default %s' % ' '.join(as_list(paths))) + + def _count_dollars_before_index(self, s: str, i: int) -> int: + """Returns the number of '$' characters right in front of s[i].""" + dollar_count = 0 + dollar_index = i - 1 + while dollar_index > 0 and s[dollar_index] == '$': + dollar_count += 1 + dollar_index -= 1 + return dollar_count + + def _line(self, text: str, indent: int = 0) -> None: + """Write 'text' word-wrapped at self.width characters.""" + leading_space = ' ' * indent + while len(leading_space) + len(text) > self.width: + # The text is too wide; wrap if possible. + + # Find the rightmost space that would obey our width constraint and + # that's not an escaped space. + available_space = self.width - len(leading_space) - len(' $') + space = available_space + while True: + space = text.rfind(' ', 0, space) + if (space < 0 or + self._count_dollars_before_index(text, space) % 2 == 0): + break + + if space < 0: + # No such space; just use the first unescaped space we can find. + space = available_space - 1 + while True: + space = text.find(' ', space + 1) + if (space < 0 or + self._count_dollars_before_index(text, space) % 2 == 0): + break + if space < 0: + # Give up on breaking. + break + + self.output.write(leading_space + text[0:space] + ' $\n') + text = text[space+1:] + + # Subsequent lines are continuations, so indent them. + leading_space = ' ' * (indent+2) + + self.output.write(leading_space + text + '\n') + + def close(self) -> None: + self.output.close() + + +def as_list(input: Optional[Union[str, List[str]]]) -> List[str]: + if input is None: + return [] + if isinstance(input, list): + return input + return [input] + + +def escape(string: str) -> str: + """Escape a string such that it can be embedded into a Ninja file without + further interpretation.""" + assert '\n' not in string, 'Ninja syntax does not allow newlines' + # We only have one special metacharacter: '$'. + return string.replace('$', '$$') + + +def expand(string: str, vars: Dict[str, str], local_vars: Dict[str, str] = {}) -> str: + """Expand a string containing $vars as Ninja would. + + Note: doesn't handle the full Ninja variable syntax, but it's enough + to make configure.py's use of it work. + """ + def exp(m: Match[str]) -> str: + var = m.group(1) + if var == '$': + return '$' + return local_vars.get(var, vars.get(var, '')) + return re.sub(r'\$(\$|\w*)', exp, string) diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 00000000..c0988481 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,2 @@ +requests +pyperclip diff --git a/tools/setup.py b/tools/setup.py new file mode 100644 index 00000000..2ffe1e27 --- /dev/null +++ b/tools/setup.py @@ -0,0 +1,50 @@ +import requests +import zipfile +import io +from pathlib import Path +import shutil +import platform + +DSD_VERSION = 'v0.1.0' + + +tools_path = Path(__file__).parent +deps_path = tools_path / 'deps' +if not deps_path.exists(): deps_path.mkdir() +root_path = tools_path.parent + + +exe_extension = '' +match platform.system(): + case 'Windows': system = 'windows'; exe_extension = '.exe' + case 'Linux': system = 'linux' + case system: + print(f'Unknown system "{system}"') + exit(1) +match platform.machine().lower(): + case 'amd64' | 'x86_64': machine = 'x86_64' + case machine: + print(f'Unknown machine: {machine}') + exit(1) + + +print('\nInstalling dsd...') +response = requests.get(f'https://github.com/AetiasHax/ds-decomp/releases/download/{DSD_VERSION}/dsd-{system}-{machine}{exe_extension}') +with open(root_path / f'dsd{exe_extension}', 'wb') as f: + f.write(response.content) + + +print('\nInstalling toolchain...') +response = requests.get('http://decomp.aetias.com/mwccarm.zip') +zip_file = zipfile.ZipFile(io.BytesIO(response.content)) +zip_file.extractall(tools_path) + + +print('\nInstalling ELFIO...') +response = requests.get('https://github.com/serge1/ELFIO/releases/download/Release_3.12/elfio-3.12.zip') +zip_file = zipfile.ZipFile(io.BytesIO(response.content)) +zip_file.extractall(deps_path) +elfio_path = deps_path / 'elfio-3.12' +elfio_new_path = deps_path / 'elfio' +if elfio_new_path.exists(): shutil.rmtree(elfio_new_path) +elfio_path.rename(elfio_new_path)