From fb63b9271dbd79492f54384084e64725676be745 Mon Sep 17 00:00:00 2001 From: Aetias Date: Tue, 10 Sep 2024 20:18:52 +0200 Subject: [PATCH] Remove non-matching functions and `progress.py` --- CONTRIBUTING.md | 29 ----- INSTALL.md | 12 -- Makefile | 4 - include/global.h | 14 +-- src/00_Core/Item/ItemManager.cpp | 6 +- tools/m2ctx.py | 3 - tools/progress.py | 205 ------------------------------- 7 files changed, 5 insertions(+), 268 deletions(-) delete mode 100644 tools/progress.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4a4ae9b8..883982ae 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,6 @@ - [Decompiling](#decompiling) - [Code style](#code-style) - [Creating new `.c`/`.cpp` files](#creating-new-ccpp-files) -- [Non-matching functions](#non-matching-functions) ## Project structure - `asm/`: Non-decompiled assembly code @@ -114,31 +113,3 @@ s32 MyClass::MyMethod(MyStruct *myStruct, s32 &anInteger) { return mInteger; } ``` - -## Non-matching functions -This project supports non-matching functions, and you can build them by using `make NONMATCHING=1`. - -Non-matching functions must be written as follows: -```cpp -#include "global.hpp" - -void NONMATCH(MyFunction)() { - #ifndef NONMATCHING - #include "../asm/path/to/asm.inc" - #else - // non-matching code here - #endif -} -``` - -When building normally, the `NONMATCH` macro will mark `MyFunction` as an assembly function, and the `NONMATCHING` macro will -not be defined so that the `asm.inc` file will be included. - -Conversely, when building in non-matching mode, `MyFunction` will be a regular C/C++ function, and the non-matching code will -be inserted instead of `asm.inc`. - -When contributing non-matching functions to this project, please build in both modes and fix any build errors you may get. -Delete the `.o` file between building in each mode so that the `Makefile` runs the compiler both times. - -> [!NOTE] -> The inline assembler does not function the same as the standalone assembler. [See differences here.](docs/inline_assembler.md#differences-from-standalone-assembler) diff --git a/INSTALL.md b/INSTALL.md index 816f3278..7e15557d 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -66,15 +66,3 @@ ARM7 BIOS in the root directory of this repository, and verify that your dumped | `arm7_bios.bin` | `6ee830c7f552c5bf194c20a2c13d5bb44bdb5c03` | Now, `make` should automatically detect the ARM7 BIOS and will build a matching ROM. - -## Building with non-matching code -Due to challenges with decompilation, some functions are not decompiled to 100% match the original assembly. Such functions are -marked with `NONMATCH` before the function declaration. - -While non-matching functions do not contribute to getting a matching ROM, they can provide useful information (e.g. updates to -structs/classes) or encourage collaboration to match the function. - -By default, non-matching functions are compiled with **inline assembly** so that the built ROM can still match the base ROM. -However, by running `make NONMATCHING=1`, non-matching functions are compiled as **C++** instead of inline assembly. - -As a result, the built ROM will not match and is not guaranteed to function identically to the base ROM. diff --git a/Makefile b/Makefile index bf37505a..c1565750 100644 --- a/Makefile +++ b/Makefile @@ -56,10 +56,6 @@ C_FLAGS := -lang=c CXX_FLAGS := -lang=c++ LD_FLAGS := -proc arm946e -nostdlib -interworking -nodead -m Entry -map closure,unused -o main.bin -msgstyle gcc -ifeq ($(NONMATCHING),1) - CC_FLAGS += -DNONMATCHING -endif - .PHONY: help help: @echo "Usage:" diff --git a/include/global.h b/include/global.h index 10a927aa..838b1732 100644 --- a/include/global.h +++ b/include/global.h @@ -7,14 +7,6 @@ #define SET_FLAG(arr, pos) ((arr)[((u32)(pos)) >> 5] |= 1 << ((pos) & 0x1f)) #define RESET_FLAG(arr, pos) ((arr)[((u32)(pos)) >> 5] &= ~(1 << ((pos) & 0x1f))) -// NONMATCH(name) marks the function `name` as nonmatching -// The reason for the macro is to easily detect it in progress.py -#ifdef NONMATCHING -#define NONMATCH(name) name -#else -#define NONMATCH(name) asm name -#endif - // KILL(name) causes a function to be excluded from the output ROM, see elfkill.cpp #define KILL(name) @@ -29,9 +21,11 @@ #define override // Define .sbss variables by using #pragma section sbss begin|end -#pragma define_section sbss ".data" ".sbss" +#pragma define_section sbss ".data" \ + ".sbss" // Force variables to be in .data by using #pragma section force_data begin|end -#pragma define_section force_data ".data" ".data" +#pragma define_section force_data ".data" \ + ".data" #endif diff --git a/src/00_Core/Item/ItemManager.cpp b/src/00_Core/Item/ItemManager.cpp index e20081d4..dccd4205 100644 --- a/src/00_Core/Item/ItemManager.cpp +++ b/src/00_Core/Item/ItemManager.cpp @@ -241,10 +241,7 @@ extern "C" bool _ZN10MapManager18func_ov00_020849f8Ei(void *param1); extern "C" bool _ZN14PlayerLinkBase18func_ov00_020bbd80Ei(unk32 param1, unk32 param2); extern "C" bool _ZNK11ItemManager7HasItemEi(); extern "C" void _ZN11ItemManager12GetEquipItemEi(); -ARM bool NONMATCH(ItemManager::func_ov00_020ad790)(unk32 param1) { - #ifndef NONMATCHING - #include "../asm/ov00/Item/ItemManager_func_ov00_020ad790.inc" - #else +ARM bool ItemManager::func_ov00_020ad790(unk32 param1) { unk32 unk1 = func_ov00_02078b40(data_027e0d38); if (unk1 == 2) return func_ov15_02136670(data_027e10a4); if (data_027e0d38->mUnk_14 == 1) return false; @@ -269,7 +266,6 @@ ARM bool NONMATCH(ItemManager::func_ov00_020ad790)(unk32 param1) { return this->GetEquipItem(equipId)->IsUsable(param1); } return false; - #endif } THUMB ShipType ItemManager::GetEquippedShipPart(ShipPart part) const { diff --git a/tools/m2ctx.py b/tools/m2ctx.py index 24e12d7a..dc43f326 100755 --- a/tools/m2ctx.py +++ b/tools/m2ctx.py @@ -34,13 +34,10 @@ def eprint(*args, **kwargs): INCLUDE_REGEX = r'^\s*#\s*include\s*([<"][\S ]+[>"])\s*$' # Finds all line comments and multiline comments COMMENT_REGEX = r'\/\/.*$|\/\*(?:.|\r|\n)+?\*\/' -# Finds all lines from #ifndef NONMATCHING to #else -NONMATCH_REGEX = r'^\s*#\s*ifndef\s*NONMATCHING\s*(?:.|\r|\n)*\n\s*#\s*else\s*$' with open(args.file, 'r') as f: contents = f.read() contents = re.sub(COMMENT_REGEX, '', contents, 0, re.MULTILINE) -contents = re.sub(NONMATCH_REGEX, '', contents, 0, re.MULTILINE) includes = re.findall(INCLUDE_REGEX, contents, re.MULTILINE) _, suffix = os.path.splitext(args.file) diff --git a/tools/progress.py b/tools/progress.py deleted file mode 100644 index c3d0935e..00000000 --- a/tools/progress.py +++ /dev/null @@ -1,205 +0,0 @@ -import sys -import os -import re -from pathlib import Path -import argparse - - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - -def remove_whitespace(s: str): - return ''.join(s.split()) - - -def get_nonmatching_funcs(source: str): - # Example: - # ARM bool NONMATCH(ItemManager::func_ov00_020ad790)(unk32 param1) - # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - funcs = re.findall(r'NONMATCH\W*\(\W*([^\)]+)', source) - return [remove_whitespace(func) for func in funcs] - - -def get_all_nonmatching_funcs(): - funcs = [] - for root, dirs, files in os.walk('src'): - for file in files: - name, ext = file.rsplit('.', 1) - if ext not in ['c', 'cpp']: continue - with open(os.path.join(root, file), 'r') as f: - source = f.read() - funcs.extend(get_nonmatching_funcs(source)) - return set(funcs) - - -class Section: - def __init__(self, name: str) -> None: - self.name = name - # Number of bytes from assembly, C/C++ and nonmatching code - # In total, they amount to the entire code - self.bytes_asm = 0 - self.bytes_c_cpp = 0 - self.bytes_nonmatch = 0 - - def total(self): - return self.bytes_asm + self.bytes_c_cpp + self.bytes_nonmatch - - def add(self, other): - self.bytes_asm += other.bytes_asm - self.bytes_c_cpp += other.bytes_c_cpp - self.bytes_nonmatch += other.bytes_nonmatch - - def print(self, indent=0): - total = self.total() - percent_asm = 100 * (self.bytes_asm / total) - percent_c_cpp = 100 * (self.bytes_c_cpp / total) - percent_nonmatch = 100 * (self.bytes_nonmatch / total) - indent = ' ' * indent - - print(f'{indent}{self.name}:') - print(f'{indent} asm = {self.bytes_asm:7} or {self.bytes_asm:6X}h bytes ({percent_asm:7.3f}%)') - print(f'{indent} C/C++ = {self.bytes_c_cpp:7} or {self.bytes_c_cpp:6X}h bytes ({percent_c_cpp:7.3f}%)') - print(f'{indent} nonmatch = {self.bytes_nonmatch:7} or {self.bytes_nonmatch:6X}h bytes ({percent_nonmatch:7.3f}%)') - print() - - -section_names = ['.text', '.rodata', '.data', '.bss', '.init'] -class Module: - def __init__(self, name: str) -> None: - self.name = name - self.sections = {name: Section(name) for name in section_names} - - def total(self): - total = Section('TOTAL') - total.bytes_asm = sum(section.bytes_asm for section in self.sections.values()) - total.bytes_c_cpp = sum(section.bytes_c_cpp for section in self.sections.values()) - total.bytes_nonmatch = sum(section.bytes_nonmatch for section in self.sections.values()) - return total - -def main(args: argparse.Namespace): - nonmatching_funcs = get_all_nonmatching_funcs() - - modules: dict[str, Module] = {} - current_module = None - - map_path = None - regions = ['eur', 'usa'] - for region in regions: - map_path = Path(f'build/{region}/main.bin.xMAP') - if map_path.is_file(): break - if map_path is None: - eprint('You must build the ROM first!') - exit(1) - - with open(f'build/{region}/main.bin.xMAP', 'r') as f: - line = '\n' - while len(line) > 0: - line = f.readline() - if line.startswith('# .'): - module_name = line[2:].strip() - current_module = Module(module_name) - modules[module_name] = current_module - - # 020AD090 00000028 .text ItemManager::Create() (ItemManager.cpp.o) - words = line.split() - if len(words) < 5: continue - - # 020AD090 - addr = words[0] - try: addr = int(addr, base=16) - except: continue - - # 00000028 - size = words[1] - try: size = int(size, base=16) - except: continue - - # .text - section = words[2] - # Much of .sbss hasn't been defined yet, so group it together with .bss - if section == '.sbss': section = '.bss' - # .ctor and .init are coupled together - if section == '.ctor': section = '.init' - section = current_module.sections[section] - - # ItemManager::Create() - func = words[3] - is_section = func.startswith('.') - # ItemManager::Create - func = func.split('(', 1)[0] - is_nonmatch = func in nonmatching_funcs - if is_nonmatch: nonmatching_funcs.remove(func) - - # (ItemManager.cpp.o) - file = words[-1] - if not file.startswith('('): continue - if not file.endswith(')'): continue - # ItemManager.cpp.o - file = file[1:-1] - - is_asm = file.endswith('.s.o') - is_c_cpp = file.endswith('.c.o') or file.endswith('.cpp.o') - if not is_asm and not is_c_cpp: - eprint(f"Unknown file extension for '{file}'") - continue - - if is_asm and is_section: section.bytes_asm += size - if is_c_cpp and not is_section: - if is_nonmatch: section.bytes_nonmatch += size - else: section.bytes_c_cpp += size - - # Remove empty sections - for module in modules.values(): - module.sections = {name: section for name, section in module.sections.items() if section.total() > 0} - # Remove empty modules - modules = {name: module for name, module in modules.items() if len(module.sections) > 0} - - if len(nonmatching_funcs) > 0: - eprint('Some nonmatching functions were not found in the map file:') - eprint('\n'.join(list(nonmatching_funcs))) - eprint() - - if args.all: - if args.sections: - sections = {name: Section(name) for name in section_names} - for module in modules.values(): - for name, section in module.sections.items(): - sections[name].add(section) - for section in sections.values(): - section.print() - if args.total: - module_totals = [module.total() for module in modules.values()] - total = Section('TOTAL') - total.bytes_asm = sum(module_total.bytes_asm for module_total in module_totals) - total.bytes_c_cpp = sum(module_total.bytes_c_cpp for module_total in module_totals) - total.bytes_nonmatch = sum(module_total.bytes_nonmatch for module_total in module_totals) - total.print() - - if args.modules: - for name, module in modules.items(): - print(f'Module {name}:') - if args.sections: - for section in module.sections.values(): - section.print(indent=2) - if args.total: - module.total().print(indent=2) - -parser = argparse.ArgumentParser(description='Parses an xMAP file to calculate and display decompilation progress') -parser.add_argument('--modules', help='display progress for each module', action='store_true') -parser.add_argument('--all', help='display progress for all modules combined', action='store_true') -parser.add_argument('--sections', help='display progress for each section', action='store_true') -parser.add_argument('--total', help='display progress for all section combined', action='store_true') - -args = parser.parse_args() - -if not args.modules and not args.all: - parser.print_help() - eprint("\nYou must specify at least one of --modules and --all, see above") - exit(1) -if not args.sections and not args.total: - parser.print_help() - eprint("\nYou must specify at least one of --sections and --total, see above") - exit(1) - -main(args)