Clean up and improvements to tools (#163)

* moved elf2dol

* removed postprocess.py

* removed vtables.py

* find_unused_asm.py

* removed section2cpp.py

* removed splitter/*

* fixed symbol names due to iconv file rename

* fixed problem building RELs caused by #160

* improved performance of a few python tools

* added new tool for finding conflict when not OK

* added ./tp setup

* don't install dol2asm dependecies with requirements.txt

* format and check for imports

* remove unused tools/difftools.py

* fixed ignore to include elf2dol

* fix compiler patcher

* ok-check now creates the patched compiler at mwcceppc_patched.exe

* Add new command to copy the build folder to the expected folder

* 'make clean' will now only clean main.dol stuff. (added clean_rels and clean_all)

* './tp pull-request' and './tp check' now doesn't include RELs by default. Use '--rels' to include them in the process.

* './tp remove-unused-asm --check' added, exitcode 0==no files, 1==exists files

Co-authored-by: Julgodis <>

tools/makerel.py

@@ -4,23 +4,31 @@ makerel.py - Generate .rel files from .plf files and a static binary
 
 """
 
-import click
 import sys
-import rich
-import logging
 import glob
 import os
-import libelf
-import librel
-import yaz0
 import traceback
 
 from pathlib import Path
-from collections import defaultdict
 from dataclasses import dataclass, field
 from typing import List, Set, Tuple, Dict
-from rich.logging import RichHandler
-from rich.console import Console
+
+try:
+    import libelf
+    import librel
+    import click
+    import logging
+
+    from rich.logging import RichHandler
+    from rich.console import Console
+except ImportError as e:
+    MISSING_PREREQUISITES = (
+        f"Missing prerequisite python module {e}.\n"
+        f"Run `python3 -m pip install --user -r tools/requirements.txt` to install prerequisites."
+    )
+
+    print(MISSING_PREREQUISITES, file=sys.stderr)
+    sys.exit(1)
 
 VERSION = "1.0"
 CONSOLE = Console()
@@ -29,21 +37,13 @@ logging.basicConfig(
     level="NOTSET",
     format="%(message)s",
     datefmt="[%X]",
-    handlers=[RichHandler(console=CONSOLE, rich_tracebacks=True)]
+    handlers=[RichHandler(console=CONSOLE, rich_tracebacks=True)],
 )
 
 LOG = logging.getLogger("rich")
 LOG.setLevel(logging.INFO)
 
-SECTION_MASK = {
-    ".init",
-    ".text",
-    ".ctors",
-    ".dtors",
-    ".rodata",
-    ".data",
-    ".bss"
-}
+SECTION_MASK = {".init", ".text", ".ctors", ".dtors", ".rodata", ".data", ".bss"}
 
 REL_SECTION_MASK = {
     ".rela.init",
@@ -55,17 +55,19 @@ REL_SECTION_MASK = {
     ".rela.bss",
 }
 
+
 @click.group()
 @click.version_option(VERSION)
 def makerel():
     pass
 
+
 @makerel.command(name="unresolved")
-@click.option('--debug/--no-debug')
+@click.option("--debug/--no-debug")
 @click.option("--output", "-o", default="forceactive.txt", required=True)
-@click.argument("str_paths", metavar='<ELFs>', nargs=-1)
+@click.argument("str_paths", metavar="<ELFs>", nargs=-1)
 def unresolved(debug, output, str_paths):
-    """ Generate a list of symbols which must be in the static executable (and other RELs). """
+    """Generate a list of symbols which must be in the static executable (and other RELs)."""
 
     if debug:
         LOG.setLevel(logging.DEBUG)
@@ -73,8 +75,7 @@ def unresolved(debug, output, str_paths):
     static, plfs = load_elfs(str_paths)
 
     if static:
-        LOG.error(
-            f"unresolved does not handle executable files '{static.path}'")
+        LOG.error(f"unresolved does not handle executable files '{static.path}'")
         sys.exit(1)
 
     undef_symbols = set()
@@ -97,15 +98,21 @@ def unresolved(debug, output, str_paths):
 
 
 @makerel.command(name="build")
-@click.option('--debug/--no-debug')
-@click.option('--yaz0', '-y', 'compress_yaz0')
-@click.option("--id-offset", '-i', 'rel_id_offset', default=1)
-@click.option("--spoof-path", '-q', 'spoof_path', default="D:\\zeldaGC_USA\\dolzel2\\bin\\Final\\")
-@click.option("--string-table", '-s', 'string_path', required=True)
-@click.option("--symbols", default="ELF", type=click.Choice(["ELF", "DEFS"], case_sensitive=False))
-@click.argument("str_paths", metavar='<ELFs>', nargs=-1)
-def build(debug, symbols, str_paths, rel_id_offset, compress_yaz0, spoof_path, string_path):
-    """ Build RELs files from a list of plfs files. """
+@click.option("--debug/--no-debug")
+@click.option("--yaz0", "-y", "compress_yaz0")
+@click.option("--id-offset", "-i", "rel_id_offset", default=1)
+@click.option(
+    "--spoof-path", "-q", "spoof_path", default="D:\\zeldaGC_USA\\dolzel2\\bin\\Final\\"
+)
+@click.option("--string-table", "-s", "string_path", required=True)
+@click.option(
+    "--symbols", default="ELF", type=click.Choice(["ELF", "DEFS"], case_sensitive=False)
+)
+@click.argument("str_paths", metavar="<ELFs>", nargs=-1)
+def build(
+    debug, symbols, str_paths, rel_id_offset, compress_yaz0, spoof_path, string_path
+):
+    """Build RELs files from a list of plfs files."""
 
     if debug:
         LOG.setLevel(logging.DEBUG)
@@ -115,7 +122,7 @@ def build(debug, symbols, str_paths, rel_id_offset, compress_yaz0, spoof_path, s
         LOG.error(f"static executable ('main.elf') expected")
         sys.exit(1)
 
-    # 
+    #
     id = rel_id_offset
     elfs = []
     for plf in plfs:
@@ -123,7 +130,7 @@ def build(debug, symbols, str_paths, rel_id_offset, compress_yaz0, spoof_path, s
             rel = IndexedElf(0, plf)
         else:
             rel = IndexedElf(id, plf)
-            id += 1 
+            id += 1
         elfs.append(rel)
 
     # sort relocations
@@ -132,7 +139,7 @@ def build(debug, symbols, str_paths, rel_id_offset, compress_yaz0, spoof_path, s
             elf._unresolved = elf.plf.symbol_map["_unresolved"][0]
 
         for _, relocations in elf.plf.section_relocations:
-                relocations.sort(key=lambda r: r.offset)
+            relocations.sort(key=lambda r: r.offset)
 
     # symbol table
     symbol_table = dict()
@@ -155,17 +162,30 @@ def build(debug, symbols, str_paths, rel_id_offset, compress_yaz0, spoof_path, s
 
     string_list.write()
 
+
 def apply_rel24_relocation(relocation, section, symbol):
     if not symbol or not isinstance(symbol, libelf.OffsetSymbol):
         return False
 
     try:
-        if librel.apply_relocation(relocation.type, 0, section.data, 0, relocation.offset, symbol.offset, relocation.addend):
+        if librel.apply_relocation(
+            relocation.type,
+            0,
+            section.data,
+            0,
+            relocation.offset,
+            symbol.offset,
+            relocation.addend,
+        ):
             return True
     except librel.RELRelocationException as e:
         LOG.error(f"applying relocation failed!")
-        LOG.error(f"relocation: {librel.RELOCATION_NAMES[relocation.type]} {relocation.offset:04X}")
-        LOG.error(f"section: {section.header.sh_addr:08X} {section.header.sh_size:04X} {section.name}")
+        LOG.error(
+            f"relocation: {librel.RELOCATION_NAMES[relocation.type]} {relocation.offset:04X}"
+        )
+        LOG.error(
+            f"section: {section.header.sh_addr:08X} {section.header.sh_size:04X} {section.name}"
+        )
         LOG.error(f"symbol: {symbol.offset:08X} {symbol.name}+0x{relocation.addend:X}")
         LOG.error(e)
         CONSOLE.print_exception()
@@ -175,6 +195,7 @@ def apply_rel24_relocation(relocation, section, symbol):
 
     return False
 
+
 @dataclass
 class StringList:
     output_path: str
@@ -189,9 +210,9 @@ class StringList:
         self.data += name
         return offset, len(name)
 
-    def write(self):   
+    def write(self):
         if len(self.data) > 0:
-            with open(self.output_path, 'w') as file:
+            with open(self.output_path, "w") as file:
                 file.write(self.data)
 
 
@@ -200,7 +221,7 @@ class ImpTable:
     id: int
     last_section: int
     section_offset: int = 0
-    relocations: List[Tuple[int,int,int,int]] = field(default_factory=list)
+    relocations: List[Tuple[int, int, int, int]] = field(default_factory=list)
     rel_offset: int = 0
 
     def section(self, section_id):
@@ -224,7 +245,7 @@ class IndexedElf:
     plf: libelf.Object
 
     _unresolved: libelf.Symbol = None
-    imp_tables: Dict[int,ImpTable] = field(default_factory=dict)
+    imp_tables: Dict[int, ImpTable] = field(default_factory=dict)
     imp_table_order: List[int] = field(default_factory=list)
     complete_relocations: Set[libelf.Relocation] = field(default_factory=set)
 
@@ -250,8 +271,7 @@ class IndexedElf:
             if replace_symbol:
                 my_symbol = replace_symbol
 
-
-        #LOG.info(f"relocation for: {my_symbol.name} ({type(my_symbol).__name__})")
+        # LOG.info(f"relocation for: {my_symbol.name} ({type(my_symbol).__name__})")
         other = None
         if isinstance(my_symbol, libelf.UndefSymbol):
             if my_symbol.name in symbol_table:
@@ -291,14 +311,20 @@ class IndexedElf:
 
                 found_section = None
                 for section in other.plf.sections.values():
-                    if ext_symbol.address >= section.header.sh_addr and ext_symbol.address < section.header.sh_addr + section.header.sh_size:
+                    if (
+                        ext_symbol.address >= section.header.sh_addr
+                        and ext_symbol.address
+                        < section.header.sh_addr + section.header.sh_size
+                    ):
                         found_section = section
                         break
 
                 if found_section:
                     ext_symbol.section = found_section
                 else:
-                    LOG.error(f"error no-section provided for relocation of: {my_symbol.name} ({type(my_symbol).__name__}) ({self.plf.name} <- {other.plf.name})")
+                    LOG.error(
+                        f"error no-section provided for relocation of: {my_symbol.name} ({type(my_symbol).__name__}) ({self.plf.name} <- {other.plf.name})"
+                    )
                     LOG.error(vars(relocation))
                     LOG.error(ext_symbol)
                     k = vars(ext_symbol)
@@ -317,8 +343,8 @@ class IndexedElf:
             section_id = ext_symbol.section.header.sh_info
             if section_id == 0:
                 section_id = ext_symbol.section.header.id
- 
-            table.relocation(relative_offset, relocation.type, section_id, addend)    
+
+            table.relocation(relative_offset, relocation.type, section_id, addend)
             self.complete_relocations.add(relocation)
             return True
         return False
@@ -333,26 +359,30 @@ class IndexedElf:
                     LOG.error(f"relocation failed: {name:<14} {relocation.symbol.name}")
                     sys.exit(1)
 
-def align_next(offset, alignment):
-    return  (offset - 1 + alignment) & ~(alignment - 1)
 
-def write_rel(path: Path, 
-        id: int, 
-        align: int, 
-        bss_align: int, 
-        bss_size: int, 
-        name_offset: int, 
-        name_size: int, 
-        prolog: libelf.Symbol,
-        epilog: libelf.Symbol,
-        unresolved: libelf.Symbol,
-        sections: List[librel.Section], 
-        imp_tables: List[ImpTable]):
+def align_next(offset, alignment):
+    return (offset - 1 + alignment) & ~(alignment - 1)
+
+
+def write_rel(
+    path: Path,
+    id: int,
+    align: int,
+    bss_align: int,
+    bss_size: int,
+    name_offset: int,
+    name_size: int,
+    prolog: libelf.Symbol,
+    epilog: libelf.Symbol,
+    unresolved: libelf.Symbol,
+    sections: List[librel.Section],
+    imp_tables: List[ImpTable],
+):
 
     output = librel.REL()
     output.index = id
     output.numSections = len(sections)
-    output.sectionInfoOffset = 0x4C # for version 3
+    output.sectionInfoOffset = 0x4C  # for version 3
     output.nameOffset = name_offset
     output.nameSize = name_size
     output.version = 3
@@ -382,7 +412,7 @@ def write_rel(path: Path,
 
     assert output.version >= 3
 
-    with path.open('wb') as file:
+    with path.open("wb") as file:
         librel.write_header(file, output)
 
         sections_offset = file.tell()
@@ -396,7 +426,7 @@ def write_rel(path: Path,
             if section.data:
                 padding = section.offset - file.tell()
                 if padding > 0:
-                    file.write(b'\x00' * padding)
+                    file.write(b"\x00" * padding)
 
                 assert section.offset == file.tell()
                 librel.write_section_data(file, section)
@@ -404,9 +434,9 @@ def write_rel(path: Path,
         output.impOffset = file.tell()
         output.impSize = len(imp_tables) * 0x8
 
-        file.write(b'\xFF' * output.impSize)
+        file.write(b"\xFF" * output.impSize)
 
-        output.fixSize =  file.tell() 
+        output.fixSize = file.tell()
         output.relOffset = file.tell()
         rel_offset = output.relOffset
         for table in imp_tables:
@@ -425,7 +455,6 @@ def write_rel(path: Path,
         librel.write_header(file, output)
 
 
-
 def write_rel_from_elf(elf: IndexedElf, string_list: StringList, compress_yaz0: bool):
     assert elf.id != 0
 
@@ -442,7 +471,7 @@ def write_rel_from_elf(elf: IndexedElf, string_list: StringList, compress_yaz0:
             table.end()
 
     # count sections
-    section_count = 1 # null section
+    section_count = 1  # null section
     for elf_section in elf.plf.sections.values():
         if elf_section.name == ".dead" or elf_section.name == ".rela.dead":
             continue
@@ -460,7 +489,9 @@ def write_rel_from_elf(elf: IndexedElf, string_list: StringList, compress_yaz0:
         if not elf_section.name in SECTION_MASK:
             continue
 
-        section = librel.Section(elf_section.header.id, 0, False, False, elf_section.header.sh_size)
+        section = librel.Section(
+            elf_section.header.id, 0, False, False, elf_section.header.sh_size
+        )
         if elf_section.header.sh_type == libelf.SHT_NOBITS:
             if elf_section.header.sh_addralign >= 1:
                 if elf_section.header.sh_addralign >= bss_align:
@@ -474,12 +505,12 @@ def write_rel_from_elf(elf: IndexedElf, string_list: StringList, compress_yaz0:
 
                 if elf_section.header.sh_addralign >= align:
                     align = elf_section.header.sh_addralign
-            
+
             section.offset = offset
             section.data = elf_section.data
             if (elf_section.header.sh_flags & libelf.SHF_EXECINSTR) != 0:
-              section.executable_flag = True
-            
+                section.executable_flag = True
+
             offset += section.length
 
         sections.append(section)
@@ -510,7 +541,21 @@ def write_rel_from_elf(elf: IndexedElf, string_list: StringList, compress_yaz0:
         tables.append(elf.imp_tables[0])
 
     # write the rel files
-    write_rel(path, elf.id, align, bss_align, bss_size, name_offset, name_size, prolog, epilog, unresolved, sections, tables)
+    write_rel(
+        path,
+        elf.id,
+        align,
+        bss_align,
+        bss_size,
+        name_offset,
+        name_size,
+        prolog,
+        epilog,
+        unresolved,
+        sections,
+        tables,
+    )
+
 
 def load_elfs(str_paths):
     static = None
@@ -535,7 +580,6 @@ def load_elfs(str_paths):
                 LOG.error(f"error: '{path}'")
                 LOG.error(e)
 
-
     return static, plfs