From a1a7cde91c4f48fd9bd49d4369b45527bd3ad4c9 Mon Sep 17 00:00:00 2001 From: Aetias Date: Sun, 28 Apr 2024 10:40:15 +0200 Subject: [PATCH 1/7] Update `CC_FLAGS` --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 004e8b16..5c69c0ff 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,8 @@ LCF_FILE := $(ROOT)/$(BUILD_DIR)/arm9_linker_script.lcf OBJS_FILE := $(ROOT)/$(BUILD_DIR)/arm9_objects.txt ASM_FLAGS := -proc arm5te -d $(REGION) -i asm -msgstyle gcc -CC_FLAGS := -proc arm946e -interworking -O4,p -enum int -i include -i- -i libs/c/include -i libs/cpp/include -nolink -d $(REGION) -char signed -sym on -msgstyle gcc -C_FLAGS := -lang=c +CC_FLAGS := -O4,p -enum int -char signed -proc arm946e -gccext,on -fp soft -inline on,noauto -Cpp_exceptions off -RTTI off -interworking -sym on -gccinc -i include -i libs/c/include -i libs/cpp/include -nolink -d $(REGION) -msgstyle gcc +C_FLAGS := -lang=c CXX_FLAGS := -lang=c++ LD_FLAGS := -proc arm946e -nostdlib -interworking -nodead -m Entry -map closure,unused -o main.bin -msgstyle gcc From 8d89e1878071b767db34da4831f3c57983064006 Mon Sep 17 00:00:00 2001 From: Aetias Date: Sun, 28 Apr 2024 10:40:34 +0200 Subject: [PATCH 2/7] Place `.cpp` ahead of corresponding `.s` files in LCF --- tools/lcf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/lcf.py b/tools/lcf.py index 35763449..869f1cc5 100644 --- a/tools/lcf.py +++ b/tools/lcf.py @@ -13,8 +13,8 @@ def name(path: str): return path.split("/")[-1] ARM9_OBJECTS = [ 'asm/main/main_02000000.s', - 'asm/main/Actor/ActorType.s', 'src/Main/Actor/ActorType.cpp', + 'asm/main/Actor/ActorType.s', 'asm/main/main_0203e8a0.s', @@ -74,13 +74,13 @@ ov00 = Overlay(name='ov00', after='ARM9', objects=[ 'asm/ov00/ov00_020b1498.s', - 'asm/ov00/Actor/Actor.s', 'src/00_Core/Actor/Actor.cpp', + 'asm/ov00/Actor/Actor.s', 'asm/ov00/ov00_020c3348.s', - 'asm/ov00/Actor/ActorManager.s', 'src/00_Core/Actor/ActorManager.cpp', + 'asm/ov00/Actor/ActorManager.s', 'asm/ov00/ov00_020c3e54.s', ]) From a7b55a988347c64c2ac612d4ac3f738ad7473ef8 Mon Sep 17 00:00:00 2001 From: Aetias Date: Sun, 28 Apr 2024 10:40:46 +0200 Subject: [PATCH 3/7] Use `argparse` in `m2ctx` --- tools/m2ctx.py | 81 ++++++++++++++------------------------------------ 1 file changed, 23 insertions(+), 58 deletions(-) diff --git a/tools/m2ctx.py b/tools/m2ctx.py index 71562c0a..7bb5885a 100755 --- a/tools/m2ctx.py +++ b/tools/m2ctx.py @@ -5,6 +5,15 @@ import pyperclip import subprocess import os from pathlib import Path +import argparse + +parser = argparse.ArgumentParser(description="Generates a context for decomp.me") +parser.add_argument('file', help="Input file to preprocess") +parser.add_argument('-f', type=str, dest='out_file', required=False, help='Output context file') +parser.add_argument('-c', action=argparse.BooleanOptionalAction, dest='clipboard', required=False, help='Copy output to clipboard') +parser.add_argument('-e', type=str, dest='encoding', required=False, default="Shift-JIS", help='Input file encoding') +parser.add_argument('-v', action=argparse.BooleanOptionalAction, dest='verbose', required=False, help='Verbose error output') +args = parser.parse_args() CXX_FLAGS = [ '-nostdinc', @@ -15,63 +24,19 @@ CXX_FLAGS = [ script_dir = Path(os.path.dirname(os.path.realpath(__file__))) root_dir = script_dir / ".." -program = os.path.basename(sys.argv[0]) - def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) -def print_usage(): - eprint(f"Usage: {program} INFILE [-f OUTFILE] [-c] [-e ENCODING] [-v]") - eprint(" INFILE \tInput file to preprocess") - eprint(" -f OUTFILE \tOutput context file") - eprint(" -c \tCopy output to clipboard") - eprint(" -e ENCODING\tInput file encoding (Default: Shift-JIS)") - eprint(" -v \tVerbose error output") - -if len(sys.argv) == 1: - print_usage() - exit(1) - - -in_file = None -out_file = None -clipboard = False -encoding = "Shift-JIS" -verbose = False -i = 1 -while i < len(sys.argv): - arg = sys.argv[i] - if arg == "-f": - i += 1 - if i >= len(sys.argv): - eprint("Expected output file after -f") - exit(1) - out_file = sys.argv[i] - elif arg == "-c": - clipboard = True - elif arg == "-e": - i += 1 - if i >= len(sys.argv): - eprint("Expected input file encoding after -e") - exit(1) - encoding = sys.argv[i] - elif arg == "-v": - verbose = True - elif arg.startswith("-"): - eprint(f"Unknown option '{arg}'") - elif in_file is None: - in_file = arg - else: - eprint(f"Duplicate input file '{arg}'") - exit(1) - i += 1 - - try: - ctx: str = subprocess.check_output(['gcc', '-E', '-P', '-undef', '-dD', *CXX_FLAGS, in_file], cwd=root_dir, encoding=encoding) + ctx: str = subprocess.check_output([ + 'gcc', + '-E', '-P', '-fworking-directory', '-undef', '-dD', + *CXX_FLAGS, + args.file + ], cwd=root_dir, encoding=args.encoding) except subprocess.CalledProcessError as e: - eprint(f"Failed to preprocess '{in_file}'") - if verbose: eprint(e) + eprint(f"Failed to preprocess '{args.file}'") + if args.verbose: eprint(e) else: eprint("Use -v for more verbose error output") exit(1) @@ -85,17 +50,17 @@ for i in reversed(range(len(lines))): ctx = ''.join(lines) -if out_file: +if args.out_file: try: - with open(out_file, "w") as file: + with open(args.out_file, "w") as file: file.write(ctx) except OSError as e: - eprint(f"Failed to write file '{out_file}'") - if verbose: eprint(e) + eprint(f"Failed to write file '{args.out_file}'") + if args.verbose: eprint(e) else: eprint("Use -v for more verbose error output") exit(1) -if clipboard: +if args.clipboard: pyperclip.copy(ctx) eprint("Copied context to clipboard") -if out_file is None and not clipboard: +if args.out_file is None and not args.clipboard: print(ctx) From e34d85431b1b7527b7054dfb2ff90117e1ed74ff Mon Sep 17 00:00:00 2001 From: Aetias Date: Sun, 28 Apr 2024 10:47:43 +0200 Subject: [PATCH 4/7] Update `decompiling.md` --- docs/decompiling.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/decompiling.md b/docs/decompiling.md index bedb1cbb..e0ea60f5 100644 --- a/docs/decompiling.md +++ b/docs/decompiling.md @@ -22,7 +22,7 @@ again. Remember to make a pull request of any notable progress you made on the s Say you've found a function you want to decompile. Here are the steps we recommend for decompiling it: 1. Visit [decomp.me](https://decomp.me/) and start decomping. 1. Under the platforms, select "Nintendo DS". -1. Select compiler version `2.0sp1p5`. +1. Select compiler preset "Phantom Hourglass". 1. Copy and paste the target assembly for your function, including the `func_start` and `func_end` macros, and the pool constants. For example: ```arm @@ -36,13 +36,9 @@ func_ov09_0211bf48: ; 0x0211bf48 thumb_func_end func_ov09_0211bf48 _0211bf50: .word data_ov09_0211f59c ``` -6. Run `m2ctx.py include/MyHeader.hpp -c` to generate a context and put it in your clipboard. +5. Run `m2ctx.py include/MyHeader.hpp -c` to generate a context and put it in your clipboard. - If no suitable header file exists, make a new one and put any structs and types you need in there. 1. Paste the context into decomp.me, and create the scratch. -1. Copy the `CC_FLAGS` from `Makefile` into the arguments field in decomp.me. - - Also copy `CXX_FLAGS` if your function comes from a `.cpp` file. Otherwise, copy `C_FLAGS` if it's a `.c` file. - - Replace the `-d $(REGION)` flag with whichever region you intend to decompile for. You can also delete the flag entirely - if the function contains no region differences. 1. Decompile the function and try to get a 100% match. - There's no ARM decompiler in decomp.me yet, but Ghidra does the job quite well. See [the Ghidra section](#the-ghidra-project) for more info. @@ -70,6 +66,18 @@ is 12 (`0xc`) bytes long and is also implicit, so we don't need to define it our > [!IMPORTANT] > An important thing to keep in mind is that a static initializer can construct multiple global objects. +## Decompiling data +> [!NOTE] +> Under construction! It's not fully clear how data is decompiled, as the compiler is strict on how it orders global variables. +> Feel free to contribute to this section or provide us with more information! + +Other than `.text` and `.init` which contain code, there are the following sections for data: +- `.rodata`: Global or static constants +- `.data`: Global or static variables +- `.bss`/`.sbss`: Global or static uninitialized variables + +You can see examples of these data sections in the [compilation section in `build_system.md`](/docs/build_system.md#compiling-code). + ## The Ghidra project We use a shared Ghidra project to analyze the game and decompile functions. To gain access to the project, install [Ghidra version 10.2.3](https://github.com/NationalSecurityAgency/ghidra/releases/tag/Ghidra_10.2.3_build) and request access From 79ac250901c572eafb64102dbc6860c8d3c3212e Mon Sep 17 00:00:00 2001 From: Aetias Date: Fri, 3 May 2024 15:16:25 +0200 Subject: [PATCH 5/7] Add `-str noreuse` compiler flag --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5c69c0ff..0dc3c52a 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ LCF_FILE := $(ROOT)/$(BUILD_DIR)/arm9_linker_script.lcf OBJS_FILE := $(ROOT)/$(BUILD_DIR)/arm9_objects.txt ASM_FLAGS := -proc arm5te -d $(REGION) -i asm -msgstyle gcc -CC_FLAGS := -O4,p -enum int -char signed -proc arm946e -gccext,on -fp soft -inline on,noauto -Cpp_exceptions off -RTTI off -interworking -sym on -gccinc -i include -i libs/c/include -i libs/cpp/include -nolink -d $(REGION) -msgstyle gcc +CC_FLAGS := -O4,p -enum int -char signed -str noreuse -proc arm946e -gccext,on -fp soft -inline on,noauto -Cpp_exceptions off -RTTI off -interworking -sym on -gccinc -i include -i libs/c/include -i libs/cpp/include -nolink -d $(REGION) -msgstyle gcc C_FLAGS := -lang=c CXX_FLAGS := -lang=c++ LD_FLAGS := -proc arm946e -nostdlib -interworking -nodead -m Entry -map closure,unused -o main.bin -msgstyle gcc From 6bce0e739d89da2f75149e4cab11393866b7427e Mon Sep 17 00:00:00 2001 From: Aetias Date: Fri, 3 May 2024 15:35:15 +0200 Subject: [PATCH 6/7] Add advice on decompiling source files --- docs/decompiling.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/decompiling.md b/docs/decompiling.md index e0ea60f5..3ff23f2b 100644 --- a/docs/decompiling.md +++ b/docs/decompiling.md @@ -2,6 +2,7 @@ This document describes how you can start decompiling code and contribute to the project. Feel free to ask for help if you get stuck or need assistance. - [Pick a source file](#pick-a-source-file) +- [Decompiling a source file](#decompiling-a-source-file) - [Decompiling a function](#decompiling-a-function) - [Decompiling `.init` functions](#decompiling-init-functions) - [The Ghidra project](#the-ghidra-project) @@ -18,13 +19,24 @@ If you want to unclaim the file, leave another comment so we can be certain that again. Remember to make a pull request of any notable progress you made on the source file, which can include [non-matching functions](/CONTRIBUTING.md#non-matching-functions). +## Decompiling a source file +It can be tricky to fully decompile an assembly file into a C/C++ source file, so here's some advice to make it easier: +- C/C++ code is built before assembly code + - This means you can take one function from the top of your assembly file, decompile it, and append it to the bottom of + your C/C++ file. +- Build the ROM often + - We recommend building every time you decompile a function. + - This is because functions can sometimes match in decomp.me, but not when building. + - If your ROM doesn't match, it's easier to know which function is wrong if it's the only function added since the last + successful build. + ## Decompiling a function Say you've found a function you want to decompile. Here are the steps we recommend for decompiling it: 1. Visit [decomp.me](https://decomp.me/) and start decomping. 1. Under the platforms, select "Nintendo DS". 1. Select compiler preset "Phantom Hourglass". -1. Copy and paste the target assembly for your function, including the `func_start` and `func_end` macros, and the pool constants. -For example: +1. Copy and paste the target assembly for your function, including the `func_start` and `func_end` macros, and the pool +constants. For example: ```arm .global func_ov09_0211bf48 thumb_func_start func_ov09_0211bf48 From 0538f141fda58bf09f2da841f2dadadb8696cdf4 Mon Sep 17 00:00:00 2001 From: Aetias Date: Fri, 3 May 2024 15:41:45 +0200 Subject: [PATCH 7/7] .gitignore `*.xMAP` --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b63315e9..a0a106e8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ ph_*/ *bios.bin /m2ctx *.sav +*.xMAP