From 161162a58a550daad8ac462a374f390b9b3aeaef Mon Sep 17 00:00:00 2001 From: Elijah Thomas Date: Wed, 4 Oct 2023 10:34:56 -0400 Subject: [PATCH] Update dtk Pulled sources from tww/echoes --- config/SOUE01/splits.txt | 25 +- config/SOUE01/symbols.txt | 48 +- configure.py | 906 ++++++++++++----------------------- tools/decompctx.py | 87 ++++ tools/download_dtk.py | 41 -- tools/download_tool.py | 93 ++++ tools/dtk_version | 1 - tools/project.py | 981 ++++++++++++++++++++++++++++++++++++++ tools/transform-dep.py | 77 --- tools/transform_dep.py | 84 ++++ 10 files changed, 1607 insertions(+), 736 deletions(-) create mode 100644 tools/decompctx.py delete mode 100644 tools/download_dtk.py create mode 100644 tools/download_tool.py delete mode 100644 tools/dtk_version create mode 100644 tools/project.py delete mode 100644 tools/transform-dep.py create mode 100644 tools/transform_dep.py diff --git a/config/SOUE01/splits.txt b/config/SOUE01/splits.txt index 60b04acc..660a6929 100644 --- a/config/SOUE01/splits.txt +++ b/config/SOUE01/splits.txt @@ -1,3 +1,18 @@ +Sections: + .init type:code + extab type:rodata + extabindex type:rodata + .text type:code + .ctors type:rodata + .dtors type:rodata + .rodata type:rodata + .data type:data + .sdata type:data + .sbss type:bss + .sdata2 type:rodata + .sbss2 type:bss + .bss type:bss + toBeSorted/save_file.cpp: .text start:0x800099B0 end:0x80009D30 @@ -21,7 +36,7 @@ d/a/obj/d_a_obj_base.cpp: d/d_base.cpp: .text start:0x80050800 end:0x80050A14 - .data start:0x80503380 end:0x805033CC + .data start:0x80503380 end:0x805033D0 .sbss start:0x805750C0 end:0x805750CC toBeSorted/sceneflag_manager.cpp: @@ -44,7 +59,7 @@ lib/rvl/CX/cx.c: lib/egg/core/eggArchive.cpp: .text start:0x80493510 end:0x80494254 - .data start:0x8056E820 end:0x8056E82C + .data start:0x8056E820 end:0x8056E830 .sbss start:0x80576728 end:0x80576730 .bss start:0x80673AC8 end:0x80673AD8 @@ -65,7 +80,7 @@ lib/egg/core/eggStream.cpp: lib/egg/core/eggAllocator.cpp: .text start:0x804952D0 end:0x804953F0 - .data start:0x8056E8D0 end:0x8056E8E4 + .data start:0x8056E8D0 end:0x8056E8E8 lib/egg/core/eggHeap.cpp: .text start:0x804953F0 end:0x80495AB0 @@ -87,7 +102,7 @@ lib/egg/core/eggAssertHeap.cpp: lib/egg/core/eggDisposer.cpp: .text start:0x80496830 end:0x80496904 - .data start:0x8056EA30 end:0x8056EA3C + .data start:0x8056EA30 end:0x8056EA40 lib/egg/core/eggThread.cpp: .text start:0x80496910 end:0x80496DD8 @@ -113,7 +128,7 @@ lib/egg/core/eggDisplay.cpp: lib/egg/core/eggColorFader.cpp: .text start:0x80497930 end:0x80497E3C - .data start:0x8056EAE8 end:0x8056EB0C + .data start:0x8056EAE8 end:0x8056EB20 .sdata2 start:0x8057F308 end:0x8057F310 lib/egg/core/eggAsyncDisplay.cpp: diff --git a/config/SOUE01/symbols.txt b/config/SOUE01/symbols.txt index 1b469cea..2cdd6e74 100644 --- a/config/SOUE01/symbols.txt +++ b/config/SOUE01/symbols.txt @@ -21824,7 +21824,7 @@ PPCMtfpscr = .text:0x803A1DC0; // type:function size:0x28 scope:global PPCMfhid2 = .text:0x803A1DF0; // type:function size:0x8 scope:global PPCMthid2 = .text:0x803A1E00; // type:function size:0x8 scope:global fn_803A1E10 = .text:0x803A1E10; // type:function size:0xC -fn_803A1E20 = .text:0x803A1E20; // type:function size:0x8 +PPCMtwpar = .text:0x803A1E20; // type:function size:0x8 scope:global PPCDisableSpeculation = .text:0x803A1E30; // type:function size:0x28 scope:global PPCSetFpNonIEEEMode = .text:0x803A1E60; // type:function size:0x8 scope:global PPCMthid4 = .text:0x803A1E70; // type:function size:0x50 scope:global @@ -27383,7 +27383,7 @@ fn_804C6340 = .text:0x804C6340; // type:function size:0x58 fn_804C63A0 = .text:0x804C63A0; // type:function size:0x13C strlen = .text:0x804C64DC; // type:function size:0x1C scope:weak __va_arg = .text:0x804C64F8; // type:function size:0xC8 scope:global -fn_804C65C0 = .text:0x804C65C0; // type:function size:0x18 +__register_global_object = .text:0x804C65C0; // type:function size:0x18 scope:global __destroy_global_chain = .text:0x804C65D8; // type:function size:0x48 scope:global fn_804C6620 = .text:0x804C6620; // type:function size:0x104 fn_804C6724 = .text:0x804C6724; // type:function size:0xBC @@ -29029,7 +29029,7 @@ lbl_804F52C8 = .rodata:0x804F52C8; // type:object size:0x40 lbl_804F5308 = .rodata:0x804F5308; // type:object size:0x10 data:4byte lbl_804F5318 = .rodata:0x804F5318; // type:object size:0x1C lbl_804F5334 = .rodata:0x804F5334; // type:object size:0x1C -__DVDDeviceErrorMessage = .rodata:0x804F5350; // type:object size:0x1C scope:global +__DVDDeviceErrorMessage = .rodata:0x804F5350; // type:object size:0x1C scope:global data:4byte lbl_804F5370 = .rodata:0x804F5370; // type:object size:0x48 data:4byte @3073 = .rodata:0x804F53B8; // type:object size:0x148 scope:local data:4byte lbl_804F5500 = .rodata:0x804F5500; // type:object size:0x1C @@ -29215,7 +29215,7 @@ lbl_804FC548 = .rodata:0x804FC548; // type:object size:0xC data:string lbl_804FC554 = .rodata:0x804FC554; // type:object size:0xC lbl_804FC560 = .rodata:0x804FC560; // type:object size:0x58 lbl_804FC5B8 = .rodata:0x804FC5B8; // type:object size:0x88 -@stringBase0 = .rodata:0x804FC640; // type:object size:0x25 scope:local +@stringBase0 = .rodata:0x804FC640; // type:object size:0x25 scope:local data:string_table lbl_804FC668 = .rodata:0x804FC668; // type:object size:0x28 data:4byte lbl_804FC690 = .rodata:0x804FC690; // type:object size:0x10 data:4byte lbl_804FC6A0 = .rodata:0x804FC6A0; // type:object size:0x200 @@ -36473,7 +36473,7 @@ lbl_8055EA90 = .data:0x8055EA90; // type:object size:0x48 lbl_8055EAD8 = .data:0x8055EAD8; // type:object size:0x48 Si = .data:0x8055EB20; // type:object size:0x14 scope:local data:4byte Type = .data:0x8055EB38; // type:object size:0x10 scope:local data:4byte -XYNTSC = .data:0x8055EB48; // type:object size:0x30 scope:local +XYNTSC = .data:0x8055EB48; // type:object size:0x30 scope:local data:byte lbl_8055EBE0 = .data:0x8055EBE0; // type:object size:0x48 lbl_8055EC28 = .data:0x8055EC28; // type:object size:0x1A4 lbl_8055EDCC = .data:0x8055EDCC; // type:object size:0xEC data:2byte @@ -36485,8 +36485,8 @@ jumptable_8055EF78 = .data:0x8055EF78; // type:object size:0x24 scope:local jumptable_8055F114 = .data:0x8055F114; // type:object size:0x24 scope:local lbl_8055F138 = .data:0x8055F138; // type:object size:0x628 lbl_8055F760 = .data:0x8055F760; // type:object size:0x170 -lbl_8055F8D0 = .data:0x8055F8D0; // type:object size:0xC0 data:4byte -lbl_8055F990 = .data:0x8055F990; // type:object size:0x10 +GXTexRegionAddrTable = .data:0x8055F8D0; // type:object size:0xC0 scope:local data:4byte +GXShutdownFuncInfo = .data:0x8055F990; // type:object size:0x10 scope:local lbl_8055F9A0 = .data:0x8055F9A0; // type:object size:0x18 lbl_8055F9B8 = .data:0x8055F9B8; // type:object size:0x18 jumptable_8055F9D0 = .data:0x8055F9D0; // type:object size:0x68 scope:local @@ -37256,7 +37256,7 @@ lbl_80570F80 = .data:0x80570F80; // type:object size:0x150 lbl_805710D0 = .data:0x805710D0; // type:object size:0x70 jumptable_80571140 = .data:0x80571140; // type:object size:0x6C scope:local lbl_805711B0 = .data:0x805711B0; // type:object size:0x40 data:4byte -@stringBase0 = .data:0x805711F0; // type:object size:0xE1 scope:local +@stringBase0 = .data:0x805711F0; // type:object size:0xE1 scope:local data:string_table lbl_805712D8 = .data:0x805712D8; // type:object size:0x20 @stringBase0 = .data:0x805712F8; // type:object size:0x1D scope:local data:string lbl_80571318 = .data:0x80571318; // type:object size:0x48 @@ -39380,7 +39380,7 @@ lbl_805749F5 = .sdata:0x805749F5; // type:object size:0x1 data:byte lbl_805749F6 = .sdata:0x805749F6; // type:object size:0x1 data:byte lbl_805749F7 = .sdata:0x805749F7; // type:object size:0x1 data:byte lbl_805749F8 = .sdata:0x805749F8; // type:object size:0x8 -lbl_80574A00 = .sdata:0x80574A00; // type:object size:0x8 data:4byte +__GXVersion = .sdata:0x80574A00; // type:object size:0x4 scope:global data:4byte lbl_80574A08 = .sdata:0x80574A08; // type:object size:0x4 lbl_80574A0C = .sdata:0x80574A0C; // type:object size:0x4 lbl_80574A10 = .sdata:0x80574A10; // type:object size:0x8 @@ -40569,15 +40569,15 @@ lbl_805761C0 = .sbss:0x805761C0; // type:object size:0x4 data:4byte lbl_805761C4 = .sbss:0x805761C4; // type:object size:0x4 data:4byte lbl_805761C8 = .sbss:0x805761C8; // type:object size:0x4 data:4byte lbl_805761CC = .sbss:0x805761CC; // type:object size:0x4 data:4byte -lbl_805761D0 = .sbss:0x805761D0; // type:object size:0x4 data:4byte -lbl_805761D4 = .sbss:0x805761D4; // type:object size:0x4 data:4byte -lbl_805761D8 = .sbss:0x805761D8; // type:object size:0x4 data:4byte -lbl_805761DC = .sbss:0x805761DC; // type:object size:0x4 data:4byte +__piReg = .sbss:0x805761D0; // type:object size:0x4 scope:global data:4byte +__cpReg = .sbss:0x805761D4; // type:object size:0x4 scope:global data:4byte +__peReg = .sbss:0x805761D8; // type:object size:0x4 scope:global data:4byte +__memReg = .sbss:0x805761DC; // type:object size:0x4 scope:global data:4byte lbl_805761E0 = .sbss:0x805761E0; // type:object size:0x8 data:4byte lbl_805761E8 = .sbss:0x805761E8; // type:object size:0x4 data:4byte lbl_805761EC = .sbss:0x805761EC; // type:object size:0x4 data:4byte lbl_805761F0 = .sbss:0x805761F0; // type:object size:0x4 data:4byte -lbl_805761F4 = .sbss:0x805761F4; // type:object size:0x4 data:4byte +@LOCAL@GXInit__FPvUl@shutdownFuncRegistered = .sbss:0x805761F4; // type:object size:0x4 scope:weak data:4byte lbl_805761F8 = .sbss:0x805761F8; // type:object size:0x1 data:byte lbl_805761F9 = .sbss:0x805761F9; // type:object size:0x1 data:byte lbl_805761FC = .sbss:0x805761FC; // type:object size:0x4 data:4byte @@ -47023,9 +47023,9 @@ lbl_8057E788 = .sdata2:0x8057E788; // type:object size:0x4 data:float lbl_8057E78C = .sdata2:0x8057E78C; // type:object size:0x4 data:float lbl_8057E790 = .sdata2:0x8057E790; // type:object size:0x4 data:float lbl_8057E794 = .sdata2:0x8057E794; // type:object size:0x4 data:float -lbl_8057E798 = .sdata2:0x8057E798; // type:object size:0x4 data:4byte -lbl_8057E79C = .sdata2:0x8057E79C; // type:object size:0x4 data:float -lbl_8057E7A0 = .sdata2:0x8057E7A0; // type:object size:0x4 data:float +__GXData = .sdata2:0x8057E798; // type:object size:0x4 scope:global data:4byte +@2712 = .sdata2:0x8057E79C; // type:object size:0x4 scope:local data:float +@2713 = .sdata2:0x8057E7A0; // type:object size:0x4 scope:local data:float lbl_8057E7A4 = .sdata2:0x8057E7A4; // type:object size:0x4 data:4byte lbl_8057E7A8 = .sdata2:0x8057E7A8; // type:object size:0x4 data:4byte lbl_8057E7AC = .sdata2:0x8057E7AC; // type:object size:0x4 data:float @@ -49148,12 +49148,12 @@ lbl_805DCA60 = .bss:0x805DCA60; // type:object size:0x1800 lbl_805DE260 = .bss:0x805DE260; // type:object size:0x8700 __OSRebootParams = .bss:0x805E6960; // type:object size:0x1C scope:global data:4byte lbl_805E6980 = .bss:0x805E6980; // type:object size:0x50 -__OSErrorTable = .bss:0x805E69D0; // type:object size:0x44 scope:global +__OSErrorTable = .bss:0x805E69D0; // type:object size:0x44 scope:global data:4byte lbl_805E6A20 = .bss:0x805E6A20; // type:object size:0x20 lbl_805E6A40 = .bss:0x805E6A40; // type:object size:0x2C8 lbl_805E6D08 = .bss:0x805E6D08; // type:object size:0x18 data:4byte Scb = .bss:0x805E6D20; // type:object size:0x54 scope:local data:2byte -DefaultThread = .bss:0x805E6D78; // type:object size:0x318 scope:local +DefaultThread = .bss:0x805E6D78; // type:object size:0x318 scope:local data:4byte lbl_805E7090 = .bss:0x805E7090; // type:object size:0x100 data:4byte lbl_805E7190 = .bss:0x805E7190; // type:object size:0x5F0 StmEhInBuf = .bss:0x805E7780; // type:object size:0x20 scope:local @@ -49170,8 +49170,8 @@ lbl_805E7AA0 = .bss:0x805E7AA0; // type:object size:0x20 data:4byte lbl_805E7AC0 = .bss:0x805E7AC0; // type:object size:0x20 lbl_805E7AE0 = .bss:0x805E7AE0; // type:object size:0x20 data:4byte lbl_805E7B00 = .bss:0x805E7B00; // type:object size:0x20 data:4byte -__OSExpireAlarm = .bss:0x805E7B20; // type:object size:0x30 scope:global -Ecb = .bss:0x805E7B50; // type:object size:0xC0 scope:local +__OSExpireAlarm = .bss:0x805E7B20; // type:object size:0x30 scope:global data:4byte +Ecb = .bss:0x805E7B50; // type:object size:0xC0 scope:local data:4byte Packet = .bss:0x805E7C10; // type:object size:0x80 scope:local data:4byte lbl_805E7C90 = .bss:0x805E7C90; // type:object size:0x20 TypeTime = .bss:0x805E7CB0; // type:object size:0x20 scope:local @@ -49181,7 +49181,7 @@ lbl_805E7EA8 = .bss:0x805E7EA8; // type:object size:0x78 data:2byte lbl_805E7F20 = .bss:0x805E7F20; // type:object size:0x58 data:2byte lbl_805E7F78 = .bss:0x805E7F78; // type:object size:0x28 data:4byte lbl_805E7FA0 = .bss:0x805E7FA0; // type:object size:0x20 data:byte -lbl_805E7FC0 = .bss:0x805E7FC0; // type:object size:0x80 +FifoObj = .bss:0x805E7FC0; // type:object size:0x80 scope:local lbl_805E8040 = .bss:0x805E8040; // type:object size:0x600 lbl_805E8640 = .bss:0x805E8640; // type:object size:0x24 data:4byte lbl_805E8664 = .bss:0x805E8664; // type:object size:0x24 data:4byte @@ -49202,7 +49202,7 @@ lbl_805EDC90 = .bss:0x805EDC90; // type:object size:0x60 WaitingQueue = .bss:0x805EDCF0; // type:object size:0x20 scope:local data:4byte lbl_805EDD20 = .bss:0x805EDD20; // type:object size:0x8C lbl_805EDDAC = .bss:0x805EDDAC; // type:object size:0xD4 -__ErrorInfo = .bss:0x805EDE80; // type:object size:0x80 scope:global +__ErrorInfo = .bss:0x805EDE80; // type:object size:0x80 scope:global data:byte CheckBuffer = .bss:0x805EDF80; // type:object size:0x20 scope:local lbl_805EDFA0 = .bss:0x805EDFA0; // type:object size:0x80 data:4byte lbl_805EE020 = .bss:0x805EE020; // type:object size:0x20 data:4byte @@ -49375,7 +49375,7 @@ lbl_80676348 = .bss:0x80676348; // type:object size:0x28 data:4byte lbl_80676370 = .bss:0x80676370; // type:object size:0x19A8 data:4byte gTRKRestoreFlags = .bss:0x80677D18; // type:object size:0x9 scope:global data:byte lbl_80677D28 = .bss:0x80677D28; // type:object size:0x18 data:4byte -gTRKSaveState = .bss:0x80677D40; // type:object size:0x94 scope:global +gTRKSaveState = .bss:0x80677D40; // type:object size:0x94 scope:global data:4byte lbl_80677DD8 = .bss:0x80677DD8; // type:object size:0x10 gTRKState = .bss:0x80677DE8; // type:object size:0xA4 scope:global data:4byte gTRKCPUState = .bss:0x80677E90; // type:object size:0x430 scope:global data:4byte diff --git a/configure.py b/configure.py index bc76fcc7..02a384e4 100644 --- a/configure.py +++ b/configure.py @@ -1,109 +1,52 @@ #!/usr/bin/env python3 -LIBS = [ - { - "lib": "SS", - "mw_version": "Wii/1.7", - "cflags": "-proc gekko -Cpp_exceptions off -RTTI off -O4,p -fp hard, -inline noauto, -enum int, -str reuse -I- -i include -i include/lib", - "host": False, - "objects": [ - ["Runtime/__init_cpp_exceptions.cpp", False], - ["toBeSorted/unk_flag_stuff.cpp", True], - ["toBeSorted/bitwise_flag_helper.cpp", True], - ["toBeSorted/sceneflag_manager.cpp", True], - ["toBeSorted/flag_space.cpp", False], - ["d/d_base.cpp", True], - ["d/a/d_a_base.cpp", False], - ["d/a/obj/d_a_obj_base.cpp", False], - ["toBeSorted/save_file.cpp", False], - ["toBeSorted/file_manager.cpp", False], - ["toBeSorted/save_manager.cpp", False], - ], - }, - # { - # "lib": "RVL", - # "mw_version": "Wii/1.7", # FIXME: Change Compiler+flags Probably - # "cflags": "-proc gekko -Cpp_exceptions off -RTTI off -O4,p -fp hard, -inline noauto, -enum int, -str reuse -I- -i include -i include/lib", - # "host": False, - # "objects": [ - # ["lib/rvl/CX/cx.c", False], - # ], - # }, - # { - # "lib": "NW4R", - # "mw_version": "Wii/1.7", # FIXME: Change Compiler+flags Probably - # "cflags": "-proc gekko -Cpp_exceptions off -RTTI off -O4,p -fp hard, -inline noauto, -enum int, -str reuse -I- -i include -i include/lib", - # "host": False, - # "objects": [ - # ], - # }, - { - "lib": "EGG", - "mw_version": "Wii/1.7", # FIXME: Change Compiler+flags Probably - "cflags": "-proc gekko -Cpp_exceptions off -RTTI off -O4,p -fp hard, -inline noauto, -enum int, -str reuse -I- -i include -i include/lib", - "host": False, - "objects": [ - ["lib/egg/core/eggArchive.cpp", False], - ["lib/egg/core/eggDvdFile.cpp", False], - ["lib/egg/core/eggDvdRipper.cpp", False], - ["lib/egg/core/eggStream.cpp", False], - ["lib/egg/core/eggAllocator.cpp", False], - ["lib/egg/core/eggHeap.cpp", False], - ["lib/egg/core/eggExpHeap.cpp", False], - ["lib/egg/core/eggFrmHeap.cpp", False], - ["lib/egg/core/eggAssertHeap.cpp", False], - ["lib/egg/core/eggDisposer.cpp", False], - ["lib/egg/core/eggThread.cpp", False], - ["lib/egg/core/eggUnk.cpp", False], - ["lib/egg/core/eggSystem.cpp", False], - ["lib/egg/core/eggDisplay.cpp", True], - ["lib/egg/core/eggColorFader.cpp", False], - ["lib/egg/core/eggAsyncDisplay.cpp", False], - ["lib/egg/core/eggVideo.cpp", False], - ["lib/egg/core/eggXfb.cpp", False], - ["lib/egg/core/eggXfbManager.cpp", False], - ["lib/egg/core/eggGraphicsFifo.cpp", False], - ["lib/egg/core/eggController.cpp", False], - ["lib/egg/math/eggMath.cpp", False], - ["lib/egg/math/eggMatrix.cpp", False], - ["lib/egg/math/eggQuat.cpp", False], - ["lib/egg/math/eggVector.cpp", False], - ["lib/egg/prim/eggAssert.cpp", False], - ], - } -] -VERSIONS = [ - "SOUE01", # 0 -] -import os -import io +### +# Generates build files for the project. +# This file also includes the project configuration, +# such as compiler flags and the object matching status. +# +# Usage: +# python3 configure.py +# ninja +# +# Append --help to see available options. +### + import sys import argparse -import json from pathlib import Path -from shutil import which -import typing -from tools import ninja_syntax +from tools.project import ( + Object, + ProjectConfig, + calculate_progress, + generate_build, + is_windows, +) +# Game versions +DEFAULT_VERSION = 0 +VERSIONS = [ + "SOUE01", # 0 + "SOUP01", # 1 +] -def path( - input: typing.Union[typing.List[Path], Path, None] -) -> typing.Optional[typing.List[str]]: - if input is None: - return None - elif isinstance(input, list): - return list(map(str, input)) - else: - return [str(input)] - +if len(VERSIONS) > 1: + versions_str = ", ".join(VERSIONS[:-1]) + f" or {VERSIONS[-1]}" +else: + versions_str = VERSIONS[0] parser = argparse.ArgumentParser() -versions_str = ", ".join(VERSIONS) +parser.add_argument( + "mode", + default="configure", + help="configure or progress (default: configure)", + nargs="?", +) parser.add_argument( "--version", dest="version", - default="SOUE01", + default=VERSIONS[DEFAULT_VERSION], help=f"version to build ({versions_str})", ) parser.add_argument( @@ -113,31 +56,17 @@ parser.add_argument( default=Path("build"), help="base build directory (default: build)", ) -parser.add_argument( - "--build-dtk", - dest="build_dtk", - type=Path, - help="path to decomp-toolkit source", -) parser.add_argument( "--compilers", dest="compilers", type=Path, - default=Path("tools/mwcc_compiler"), - help="path to compilers (default: tools/mwcc_compiler)", -) -parser.add_argument( - "--orig", - dest="orig", - type=Path, - default=Path("orig"), - help="path to retail files (default: orig)", + help="path to compilers (optional)", ) parser.add_argument( "--map", dest="map", action="store_true", - help="generate map file", + help="generate map file(s)", ) parser.add_argument( "--debug", @@ -145,493 +74,294 @@ parser.add_argument( action="store_true", help="build with debug info (non-matching)", ) -if os.name != "nt" and not "_NT-" in os.uname().sysname: +if not is_windows(): parser.add_argument( - "--wine", - dest="wine", + "--wrapper", + dest="wrapper", type=Path, - help="path to wine (or wibo)", + help="path to wibo or wine (optional)", ) +parser.add_argument( + "--build-dtk", + dest="build_dtk", + type=Path, + help="path to decomp-toolkit source (optional)", +) +parser.add_argument( + "--sjiswrap", + dest="sjiswrap", + type=Path, + help="path to sjiswrap.exe (optional)", +) +parser.add_argument( + "--verbose", + dest="verbose", + action="store_true", + help="print verbose output", +) args = parser.parse_args() -# On Windows, we need this to use && in commands -chain = "cmd /c " if os.name == "nt" else "" +config = ProjectConfig() +config.version = args.version.upper() +if config.version not in VERSIONS: + sys.exit(f"Invalid version '{config.version}', expected {versions_str}") +version_num = VERSIONS.index(config.version) +# config.build_rels = False -out = io.StringIO() -n = ninja_syntax.Writer(out) +# Apply arguments +config.build_dir = args.build_dir +config.build_dtk_path = args.build_dtk +config.compilers_path = args.compilers +config.debug = args.debug +config.generate_map = args.map +config.sjiswrap_path = args.sjiswrap +if not is_windows(): + config.wrapper = args.wrapper -n.variable("ninja_required_version", "1.3") -n.newline() +# Tool versions +config.compilers_tag = "1" +config.dtk_tag = "v0.5.5" +config.sjiswrap_tag = "v1.1.1" +config.wibo_tag = "0.5.1" -n.comment("The arguments passed to configure.py, for rerunning it.") -n.variable("configure_args", sys.argv[1:]) -n.variable("python", f'"{sys.executable}"') -n.newline() +# Project +config.config_path = Path("config") / config.version / "config.yml" +config.check_sha_path = Path("orig") / f"{config.version}.sha1" +config.linker_version = "Wii/1.7" +config.ldflags = [ + "-fp hardware", + "-nodefaults", +] -### -# Variables -### -n.comment("Variables") -version = args.version -version_num = VERSIONS.index(args.version) -build_path = args.build_dir / version -config_path = Path("config") / version / "config.yml" - -cflags_base = f"-proc gekko -nodefaults -Cpp_exceptions off -RTTI off -fp hard -fp_contract on -O4,p -maxerrors 1 -enum int -inline auto -str reuse -nosyspath -DVERSION={version_num} -i include" -if args.debug: - cflags_base += " -sym on -D_DEBUG" +# Base flags, common to most GC/Wii games. +# Generally leave untouched, with overrides added below. +cflags_base = [ + "-O4,p", + "-nodefaults", + "-proc gekko", + "-align powerpc", + "-enum int", + "-fp hardware", + "-Cpp_exceptions off", + # "-W all", + "-inline auto", + "-maxerrors 1", + "-nosyspath", + "-RTTI off", + "-fp_contract on", + "-str reuse", + # "-multibyte", + # "-I-", + "-i include", + "-i include/lib", + f"-DVERSION={version_num}", +] +if config.debug: + cflags_base.extend(["-sym on", "-DDEBUG=1"]) else: - cflags_base += " -DNDEBUG" -n.variable("cflags_base", cflags_base) -n.variable( - "cflags_retro", - "$cflags_base -use_lmw_stmw on -str reuse,pool,readonly -gccinc -inline deferred,noauto -common on", -) -n.variable( - "cflags_runtime", - "$cflags_base -use_lmw_stmw on -str reuse,pool,readonly -gccinc -inline deferred,auto", -) -n.variable("cflags_musyx", "$cflags_base -str reuse,pool,readonly -fp_contract off") -ldscript_path = build_path / "ldscript.lcf" -ldflags = f"-fp hard -nodefaults -lcf {ldscript_path}" -map_path = build_path / f"main.MAP" -if args.map: - ldflags += f" -map {map_path} -mapunused -listclosure" -if args.debug: - ldflags += " -g" -n.variable("ldflags", ldflags) -mw_link_version = "GC/3.0" -n.variable("mw_version", mw_link_version) -if os.name == "nt": - exe = ".exe" - wine = "" + cflags_base.append("-DNDEBUG=1") + +# Metrowerks library flags +cflags_runtime = [ + *cflags_base, + "-use_lmw_stmw on", + "-str reuse,pool,readonly", + "-inline deferred,auto", +] + +# Dolphin library flags +cflags_dolphin = [ + *cflags_base, + "-use_lmw_stmw on", + "-str reuse,pool,readonly", + "-inline auto", +] + +# Framework flags +cflags_framework = [ + *cflags_base, + "-inline noauto", +] + +# REL flags +cflags_rel = [ + *cflags_framework, + "-sdata 0", + "-sdata2 0", +] + + +# Helper function for single-object RELs +def Rel(status, rel_name, cpp_name, extra_cflags=[]): + return { + "lib": rel_name, + "mw_version": "Wii/1.7", + "cflags": cflags_rel + extra_cflags, + "host": True, + "objects": [ + Object(status, cpp_name), + ], + } + +# From tww. IDK if it needs changing for Wii (probably) +# Helper function for Dolphin libraries +def DolphinLib(lib_name, objects): + return { + "lib": lib_name, + "mw_version": "GC/1.2.5n", + "cflags": cflags_dolphin, # TODO check + "host": False, + "objects": objects, + } + +def EGGLib(lib_name, objects): + return { + "lib": lib_name, + "mw_version": "Wii/1.7", + "cflags": cflags_base, # TODO check + "host": False, + "objects": objects, + } + +def nw4rLib(lib_name, objects): + return { + "lib": lib_name, + "mw_version": "Wii/1.7", + "cflags": cflags_base, # TODO check + "host": False, + "objects": objects, + } + +Matching = True +NonMatching = False + +config.warn_missing_config = False +config.warn_missing_source = False # TODO +config.libs = [ + { + "lib": "framework", + "mw_version": "Wii/1.7", + "cflags": cflags_framework, + "host": False, + "objects": [ + # machine (m_name + Object(NonMatching, "Runtime/__init_cpp_exceptions.cpp"), + Object(Matching, "toBeSorted/unk_flag_stuff.cpp"), + Object(Matching, "toBeSorted/bitwise_flag_helper.cpp"), + Object(Matching, "toBeSorted/sceneflag_manager.cpp"), + Object(NonMatching, "toBeSorted/flag_space.cpp"), + Object(Matching, "d/d_base.cpp"), + Object(NonMatching, "d/a/d_a_base.cpp"), + Object(NonMatching,"d/a/obj/d_a_obj_base.cpp"), + Object(NonMatching,"toBeSorted/save_file.cpp"), + Object(NonMatching,"toBeSorted/file_manager.cpp"), + Object(NonMatching,"toBeSorted/save_manager.cpp"), + + # framework (f_name) + + # d stuff (d_name) + + ], + }, + # DolphinLib( + # "base", + # [ ], + # ), + # DolphinLib( + # "name", + # [ ], + # ), + # { + # "lib": "Runtime.PPCEABI.H", + # "mw_version": "Wii/1.7", #TODO: change + # "cflags": cflags_runtime, + # "host": False, + # "objects": [ ], + # }, + ## EGG + EGGLib( + "core", + [ + Object(NonMatching, "lib/egg/core/eggArchive.cpp"), + Object(NonMatching, "lib/egg/core/eggDvdFile.cpp"), + Object(NonMatching, "lib/egg/core/eggDvdRipper.cpp"), + Object(NonMatching, "lib/egg/core/eggStream.cpp"), + Object(NonMatching, "lib/egg/core/eggAllocator.cpp"), + Object(NonMatching, "lib/egg/core/eggHeap.cpp"), + Object(NonMatching, "lib/egg/core/eggExpHeap.cpp"), + Object(NonMatching, "lib/egg/core/eggFrmHeap.cpp"), + Object(NonMatching, "lib/egg/core/eggAssertHeap.cpp"), + Object(NonMatching, "lib/egg/core/eggDisposer.cpp"), + Object(NonMatching, "lib/egg/core/eggThread.cpp"), + Object(NonMatching, "lib/egg/core/eggUnk.cpp"), + Object(NonMatching, "lib/egg/core/eggSystem.cpp"), + Object( Matching, "lib/egg/core/eggDisplay.cpp"), + Object(NonMatching, "lib/egg/core/eggColorFader.cpp"), + Object(NonMatching, "lib/egg/core/eggAsyncDisplay.cpp"), + Object(NonMatching, "lib/egg/core/eggVideo.cpp"), + Object(NonMatching, "lib/egg/core/eggXfb.cpp"), + Object(NonMatching, "lib/egg/core/eggXfbManager.cpp"), + Object(NonMatching, "lib/egg/core/eggGraphicsFifo.cpp"), + Object(NonMatching, "lib/egg/core/eggController.cpp"), + ] + ), + EGGLib( + "math", + [ + Object(NonMatching, "lib/egg/math/eggMath.cpp"), + Object(NonMatching, "lib/egg/math/eggMatrix.cpp"), + Object(NonMatching, "lib/egg/math/eggQuat.cpp"), + Object(NonMatching, "lib/egg/math/eggVector.cpp"), + Object(NonMatching, "lib/egg/prim/eggAssert.cpp"), + ] + ), + EGGLib( + "prim", + [ + Object(NonMatching, "lib/egg/prim/eggAssert.cpp"), + ] + ), + # { + # "lib": "MSL_C", + # "mw_version": "GC/1.3.2", + # "cflags": cflags_runtime, + # "host": False, + # "objects": [ + # ], + # }, + # { + # "lib": "TRK_MINNOW_DOLPHIN", + # "mw_version": "GC/1.3.2", + # "cflags": cflags_runtime, + # "host": False, + # "objects": [ + # ], + # }, + + # Begin RELs + # { + # "lib": "REL", + # "mw_version": "GC/1.3.2", + # "cflags": cflags_rel, + # "host": False, + # "objects": [ + # Object(Matching, "REL/executor.c"), + # Object( + # Matching, + # "REL/global_destructor_chain.c", + # source="Runtime.PPCEABI.H/global_destructor_chain.c", + # ), + # ], + # }, + # Rel(Matching/NonMatching, "rel_name", "rel_folder/rel_name.cpp"), +] + +if args.mode == "configure": + # Write build.ninja and objdiff.json + generate_build(config) +elif args.mode == "progress": + # Print progress and write progress.json + config.progress_each_module = args.verbose + calculate_progress(config) else: - if "_NT-" in os.uname().sysname: - # MSYS2 - wine = "" - elif args.wine: - wine = f"{args.wine} " - elif which("wibo") is not None: - wine = "wibo " - else: - wine = "wine " - exe = "" -n.newline() - -### -# Tooling -### -n.comment("decomp-toolkit") - -tools_path = Path("tools") -build_tools_path = args.build_dir / "tools" - -if args.build_dtk: - dtk = build_tools_path / "release" / f"dtk{exe}" - n.rule( - name="cargo", - command="cargo build --release --manifest-path $in --bin $bin --target-dir $target", - description="CARGO $bin", - depfile=path(Path("$target") / "release" / "$bin.d"), - deps="gcc", - ) - n.build( - outputs=path(dtk), - rule="cargo", - inputs=path(args.build_dtk / "Cargo.toml"), - variables={ - "bin": "dtk", - "target": build_tools_path, - }, - ) -else: - dtk = build_tools_path / f"dtk{exe}" - download_dtk = tools_path / "download_dtk.py" - n.rule( - name="download_dtk", - command=f"$python {download_dtk} $in $out", - description="DOWNLOAD $out", - ) - n.build( - outputs=path(dtk), - rule="download_dtk", - inputs=path(tools_path / "dtk_version"), - implicit=path(download_dtk), - ) -n.newline() - -### -# Rules -### -compiler_path = args.compilers / "$mw_version" -mwcc = compiler_path / "mwcceppc.exe" -mwld = compiler_path / "mwldeppc.exe" - -mwcc_cmd = f"{chain}{wine}{mwcc} $cflags -MMD -c $in -o $basedir" -mwld_cmd = f"{wine}{mwld} $ldflags -o $out @$out.rsp" -ar_cmd = f"{dtk} ar create $out @$out.rsp" - -if os.name != "nt": - transform_dep = tools_path / "transform-dep.py" - transform_dep_cmd = f" && $python {transform_dep} $basefile.d $basefile.d" - mwcc_cmd += transform_dep_cmd - -n.comment("Link ELF file") -n.rule( - name="link", - command=mwld_cmd, - description="LINK $out", - rspfile="$out.rsp", - rspfile_content="$in_newline", -) -n.newline() - -n.comment("MWCC build") -n.rule( - name="mwcc", - command=mwcc_cmd, - description="MWCC $out", - depfile="$basefile.d", - deps="gcc", -) -n.newline() - -n.comment("Host build") -n.variable("host_cflags", "-I include -Wno-trigraphs") -n.variable( - "host_cppflags", - "-std=c++98 -I include -fno-exceptions -fno-rtti -D_CRT_SECURE_NO_WARNINGS -Wno-trigraphs -Wno-c++11-extensions", -) -n.rule( - name="host_cc", - command="clang $host_cflags -c -o $out $in", - description="CC $out", -) -n.rule( - name="host_cpp", - command="clang++ $host_cppflags -c -o $out $in", - description="CXX $out", -) -n.newline() - -### -# Rules for source files -### -n.comment("Source files") -build_obj_path = build_path / "obj" -build_src_path = build_path / "src" -build_host_path = build_path / "host" -build_config_path = build_path / "config.json" - -build_obj_path.mkdir(parents=True, exist_ok=True) -build_src_path.mkdir(parents=True, exist_ok=True) -objdiff_config = { - "custom_make": "ninja", - "target_dir": str(build_obj_path), - "base_dir": str(build_src_path), - "build_target": False, - "watch_patterns": [ - "*.c", - "*.cp", - "*.cpp", - "*.h", - "*.hpp", - "*.py", - "*.yml", - "*.txt", - "*.json", - ], - "units": [], -} - - -def locate_unit(unit): - for lib in LIBS: - for obj in lib["objects"]: - if obj[0] == unit: - return [lib, obj] - return None - - -has_units = False -if build_config_path.is_file(): - has_units = True - - src_path = Path("src") - link_inputs = [] - used_compiler_versions = set() - source_inputs = [] - host_source_inputs = [] - - with open(build_config_path) as r: - data = json.load(r) - for unit in data["units"]: - obj_path, unit = unit["object"], unit["name"] - unit_path = src_path / unit - if unit_path.exists(): - result = locate_unit(unit) - if result: - lib, object = result - lib_name = lib["lib"] - - completed = None - options = { - "add_to_all": True, - "mw_version": None, - "cflags": None, - } - if type(object) is list: - if len(object) > 1: - completed = object[1] - if len(object) > 2: - options.update(object[2]) - object = object[0] - - mw_version = options["mw_version"] or lib["mw_version"] - cflags = options["cflags"] or lib["cflags"] - used_compiler_versions.add(mw_version) - - n.comment(f"{unit}: {lib_name} (linked {completed})") - - base_object = Path(object).with_suffix("") - src_obj_path = build_src_path / f"{base_object}.o" - n.build( - outputs=path(src_obj_path), - rule="mwcc", - inputs=path(unit_path), - variables={ - "mw_version": mw_version, - "cflags": cflags, - "basedir": os.path.dirname( - build_src_path / f"{base_object}" - ), - "basefile": path(build_src_path / f"{base_object}"), - }, - ) - - if lib["host"]: - host_obj_path = build_host_path / f"{base_object}.o" - n.build( - outputs=path(host_obj_path), - rule="host_cc" if unit_path.suffix == ".c" else "host_cpp", - inputs=path(unit_path), - variables={ - "basedir": os.path.dirname( - build_host_path / f"{base_object}" - ), - "basefile": path(build_host_path / f"{base_object}"), - }, - ) - if options["add_to_all"]: - host_source_inputs.append(host_obj_path) - - if options["add_to_all"]: - source_inputs.append(src_obj_path) - - objdiff_config["units"].append( - { - "name": unit, - "path": f"{base_object}.o", - "reverse_fn_order": "deferred" in cflags, - } - ) - - if completed: - obj_path = src_obj_path - else: - print(f"No configuration entry found for {unit}") - exit(1) - link_inputs.append(obj_path) - n.newline() - - # Check if all compiler versions exist - for mw_version in used_compiler_versions: - mw_path = args.compilers / mw_version / "mwcceppc.exe" - if not os.path.exists(mw_path): - print(f"Compiler {mw_path} does not exist") - exit(1) - - # Check if linker exists - mw_path = args.compilers / mw_link_version / "mwldeppc.exe" - if not os.path.exists(mw_path): - print(f"Linker {mw_path} does not exist") - exit(1) - - ### - # Link - ### - n.comment("Link") - elf_path = build_path / "main.elf" - if args.map: - n.build( - outputs=path(elf_path), - rule="link", - inputs=path(link_inputs), - implicit=path(ldscript_path), - implicit_outputs=path(map_path), - ) - else: - n.build( - outputs=path(elf_path), - rule="link", - inputs=path(link_inputs), - implicit=path(ldscript_path), - ) - n.newline() - - ### - # Helper rule for building all source files - ### - n.comment("Build all source files") - n.build( - outputs="all_source", - rule="phony", - inputs=path(source_inputs), - ) - n.newline() - - ### - # Helper rule for building all source files, with a host compiler - ### - n.comment("Build all source files with a host compiler") - n.build( - outputs="all_source_host", - rule="phony", - inputs=path(host_source_inputs), - ) - n.newline() - - ### - # Generate DOL - ### - n.comment("Generate DOL") - dol_path = build_path / "main.dol" - n.rule( - name="elf2dol", - command=f"{dtk} elf2dol $in $out", - description="DOL $out", - ) - n.build( - outputs=path(dol_path), - rule="elf2dol", - inputs=path(elf_path), - implicit=path(dtk), - ) - n.newline() - - ### - # Check DOL hash - ### - n.comment("Check DOL hash") - dol_ok_path = str(dol_path) + ".ok" - n.rule( - name="check", - command=f"{dtk} shasum -c $in -o $out", - description="CHECK $in", - ) - n.build( - outputs=path(dol_ok_path), - rule="check", - inputs=path(Path("orig") / f"{version}.sha1"), - implicit=path([dol_path, dtk]), - ) - n.newline() - - ### - # Helper tools - ### - n.comment("Check for mismatching symbols") - n.rule( - name="dol_diff", - command=f"{dtk} -L error dol diff $in", - description=f"DIFF {elf_path}", - ) - n.build( - inputs=path([config_path, elf_path, map_path]), - outputs="dol_diff", - rule="dol_diff", - ) - n.build( - outputs="diff", - rule="phony", - inputs="dol_diff", - ) - n.newline() - - n.comment("Apply symbols from linked ELF") - n.rule( - name="dol_apply", - command=f"{dtk} dol apply $in", - description=f"APPLY {elf_path}", - ) - n.build( - inputs=path([config_path, elf_path, map_path]), - outputs="dol_apply", - rule="dol_apply", - implicit=path([dol_ok_path]) - ) - n.build( - outputs="apply", - rule="phony", - inputs="dol_apply", - ) - -### -# DOL split -### -n.comment("Generate objects from original DOL") -n.rule( - name="split", - command=f"{dtk} dol split $in $out_dir", - description="SPLIT $in", - depfile="$out_dir/dep", - deps="gcc", -) -n.build( - inputs=path(config_path), - outputs=path(build_config_path), - rule="split", - implicit=path(dtk), - variables={"out_dir": path(build_path)}, -) -n.newline() - -### -# Regenerate on change -### -n.comment("Reconfigure on change") -script = os.path.relpath(__file__) -n.rule( - name="configure", - command=f"$python {script} $configure_args", - generator=True, - description=f"RUN {script}", -) -n.build( - outputs="build.ninja", - rule="configure", - implicit=path([script, tools_path / "ninja_syntax.py", build_config_path]), -) -n.newline() - -### -# Default rule -### -n.comment("Default rule") -if has_units: - n.default(path(dol_ok_path)) -else: - n.default(path(build_config_path)) - -### -# Write build.ninja -### -with open("build.ninja", "w") as f: - f.write(out.getvalue()) -n.close() - -### -# Write objdiff config -### -with open("objdiff.json", "w") as w: - json.dump(objdiff_config, w, indent=4) + sys.exit("Unknown mode: " + args.mode) \ No newline at end of file diff --git a/tools/decompctx.py b/tools/decompctx.py new file mode 100644 index 00000000..c42b1828 --- /dev/null +++ b/tools/decompctx.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +### +# Generates a ctx.c file, usable for "Context" on https://decomp.me. +# +# Usage: +# python3 tools/decompctx.py src/file.cpp +# +# If changes are made, please submit a PR to +# https://github.com/encounter/dtk-template +### + +import argparse +import os +import re + +script_dir = os.path.dirname(os.path.realpath(__file__)) +root_dir = os.path.abspath(os.path.join(script_dir, "..")) +src_dir = os.path.join(root_dir, "src") +include_dir = os.path.join(root_dir, "include") + +include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]$') +guard_pattern = re.compile(r'^#ifndef\s+(.*)$') + +defines = set() + +def import_h_file(in_file: str, r_path: str) -> str: + rel_path = os.path.join(root_dir, r_path, in_file) + inc_path = os.path.join(include_dir, in_file) + if os.path.exists(rel_path): + return import_c_file(rel_path) + elif os.path.exists(inc_path): + return import_c_file(inc_path) + else: + print("Failed to locate", in_file) + exit(1) + +def import_c_file(in_file) -> str: + in_file = os.path.relpath(in_file, root_dir) + out_text = '' + + try: + with open(in_file, encoding="utf-8") as file: + out_text += process_file(in_file, list(file)) + except Exception: + with open(in_file) as file: + out_text += process_file(in_file, list(file)) + return out_text + +def process_file(in_file: str, lines) -> str: + out_text = '' + for idx, line in enumerate(lines): + guard_match = guard_pattern.match(line.strip()) + if idx == 0: + if guard_match: + if guard_match[1] in defines: + break + defines.add(guard_match[1]) + print("Processing file", in_file) + include_match = include_pattern.match(line.strip()) + if include_match and not include_match[1].endswith(".s"): + out_text += f"/* \"{in_file}\" line {idx} \"{include_match[1]}\" */\n" + out_text += import_h_file(include_match[1], os.path.dirname(in_file)) + out_text += f"/* end \"{include_match[1]}\" */\n" + else: + out_text += line + + return out_text + +def main(): + parser = argparse.ArgumentParser( + description="""Create a context file which can be used for decomp.me""" + ) + parser.add_argument( + "c_file", + help="""File from which to create context""", + ) + args = parser.parse_args() + + output = import_c_file(args.c_file) + + with open(os.path.join(root_dir, "ctx.c"), "w", encoding="utf-8") as f: + f.write(output) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tools/download_dtk.py b/tools/download_dtk.py deleted file mode 100644 index 53273cdd..00000000 --- a/tools/download_dtk.py +++ /dev/null @@ -1,41 +0,0 @@ -import argparse -import urllib.request -import os -import stat -import platform -from pathlib import Path - -REPO = "https://github.com/encounter/decomp-toolkit" - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("tag_file", help="file containing GitHub tag") - parser.add_argument("output", type=Path, help="output file path") - args = parser.parse_args() - - with open(args.tag_file, "r") as f: - tag = f.readline().rstrip() - - uname = platform.uname() - suffix = "" - system = uname.system.lower() - if system == "darwin": - system = "macos" - elif system == "windows": - suffix = ".exe" - arch = uname.machine.lower() - if arch == "amd64": - arch = "x86_64" - - url = f"{REPO}/releases/download/{tag}/dtk-{system}-{arch}{suffix}" - output = args.output - # print(f"Downloading {url} to {output}") - urllib.request.urlretrieve(url, output) - - st = os.stat(output) - os.chmod(output, st.st_mode | stat.S_IEXEC) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/tools/download_tool.py b/tools/download_tool.py new file mode 100644 index 00000000..27099669 --- /dev/null +++ b/tools/download_tool.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 + +### +# Downloads various tools from GitHub releases. +# +# Usage: +# python3 tools/download_tool.py wibo build/tools/wibo --tag 1.0.0 +# +# If changes are made, please submit a PR to +# https://github.com/encounter/dtk-template +### + +import argparse +import io +import os +import platform +import shutil +import stat +import sys +import urllib.request +import zipfile + +from pathlib import Path + + +def dtk_url(tag): + uname = platform.uname() + suffix = "" + system = uname.system.lower() + if system == "darwin": + system = "macos" + elif system == "windows": + suffix = ".exe" + arch = uname.machine.lower() + if arch == "amd64": + arch = "x86_64" + + repo = "https://github.com/encounter/decomp-toolkit" + return f"{repo}/releases/download/{tag}/dtk-{system}-{arch}{suffix}" + + +def sjiswrap_url(tag): + repo = "https://github.com/encounter/sjiswrap" + return f"{repo}/releases/download/{tag}/sjiswrap-windows-x86.exe" + + +def wibo_url(tag): + repo = "https://github.com/decompals/wibo" + return f"{repo}/releases/download/{tag}/wibo" + + +def compilers_url(tag): + if tag == "1": + return "https://cdn.discordapp.com/attachments/727918646525165659/1129759991696457728/GC_WII_COMPILERS.zip" + else: + sys.exit("Unknown compilers tag %s" % tag) + + +TOOLS = { + "dtk": dtk_url, + "sjiswrap": sjiswrap_url, + "wibo": wibo_url, + "compilers": compilers_url, +} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("tool", help="Tool name") + parser.add_argument("output", type=Path, help="output file path") + parser.add_argument("--tag", help="GitHub tag", required=True) + args = parser.parse_args() + + url = TOOLS[args.tool](args.tag) + output = Path(args.output) + + print(f"Downloading {url} to {output}") + req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) + with urllib.request.urlopen(req) as response: + if url.endswith(".zip"): + data = io.BytesIO(response.read()) + with zipfile.ZipFile(data) as f: + f.extractall(output) + output.touch(mode=0o755) + else: + with open(output, "wb") as f: + shutil.copyfileobj(response, f) + st = os.stat(output) + os.chmod(output, st.st_mode | stat.S_IEXEC) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tools/dtk_version b/tools/dtk_version deleted file mode 100644 index 18d2649b..00000000 --- a/tools/dtk_version +++ /dev/null @@ -1 +0,0 @@ -v0.3.5 \ No newline at end of file diff --git a/tools/project.py b/tools/project.py new file mode 100644 index 00000000..cdadec68 --- /dev/null +++ b/tools/project.py @@ -0,0 +1,981 @@ +### +# decomp-toolkit project generator +# Generates build.ninja and objdiff.json. +# +# This generator is intentionally project-agnostic +# and shared between multiple projects. Any configuration +# specific to a project should be added to `configure.py`. +# +# If changes are made, please submit a PR to +# https://github.com/encounter/dtk-template +### + +import io +import json +import os +import platform +import sys + +from pathlib import Path +from . import ninja_syntax + +if sys.platform == "cygwin": + sys.exit( + f"Cygwin/MSYS2 is not supported." + f"\nPlease use native Windows Python instead." + f"\n(Current path: {sys.executable})" + ) + + +class ProjectConfig: + def __init__(self): + # Paths + self.build_dir = Path("build") + self.src_dir = Path("src") + self.tools_dir = Path("tools") + + # Tooling + self.dtk_tag = None # Git tag + self.build_dtk_path = None # If None, download + self.compilers_tag = None # 1 + self.compilers_path = None # If None, download + self.wibo_tag = None # Git tag + self.wrapper = None # If None, download wibo on Linux + self.sjiswrap_tag = None # Git tag + self.sjiswrap_path = None # If None, download + + # Project config + self.build_rels = True # Build REL files + self.check_sha_path = None # Path to version.sha1 + self.config_path = None # Path to config.yml + self.debug = False # Build with debug info + self.generate_map = False # Generate map file(s) + self.ldflags = None # Linker flags + self.libs = None # List of libraries + self.linker_version = None # mwld version + self.version = None # Version name + self.warn_missing_config = False # Warn on missing unit configuration + self.warn_missing_source = False # Warn on missing source file + + # Progress output and progress.json config + self.progress_all = True # Include combined "all" category + self.progress_modules = True # Include combined "modules" category + self.progress_each_module = ( + True # Include individual modules, disable for large numbers of modules + ) + + def validate(self): + required_attrs = [ + "build_dir", + "src_dir", + "tools_dir", + "check_sha_path", + "config_path", + "ldflags", + "linker_version", + "libs", + "version", + ] + for attr in required_attrs: + if getattr(self, attr) is None: + sys.exit(f"ProjectConfig.{attr} missing") + + def find_object(self, name): + for lib in self.libs: + for obj in lib["objects"]: + if obj.name == name: + return [lib, obj] + return None + + def out_path(self): + return self.build_dir / self.version + + +class Object: + def __init__(self, completed, name, **options): + self.name = name + self.completed = completed + self.options = { + "add_to_all": True, + "cflags": None, + "mw_version": None, + "shiftjis": True, + "source": name, + } + self.options.update(options) + + +def is_windows(): + return os.name == "nt" + + +# On Windows, we need this to use && in commands +CHAIN = "cmd /c " if is_windows() else "" +# Native executable extension +EXE = ".exe" if is_windows() else "" + + +# Replace forward slashes with backslashes on Windows +def os_str(value): + return str(value).replace("/", os.sep) + + +# Replace backslashes with forward slashes on Windows +def unix_str(value): + return str(value).replace(os.sep, "/") + + +# Stringify paths for ninja_syntax +def path(value): + if value is None: + return None + elif isinstance(value, list): + return list(map(os_str, filter(lambda x: x is not None, value))) + else: + return [os_str(value)] + + +# Load decomp-toolkit generated config.json +def load_build_config(config, build_config_path): + if not build_config_path.is_file(): + return None + + def versiontuple(v): + return tuple(map(int, (v.split(".")))) + + f = open(build_config_path, "r", encoding="utf-8") + build_config = json.load(f) + config_version = build_config.get("version") + if not config_version: + # Invalid config.json + f.close() + os.remove(build_config_path) + return None + + dtk_version = config.dtk_tag[1:] # Strip v + if versiontuple(config_version) < versiontuple(dtk_version): + # Outdated config.json + f.close() + os.remove(build_config_path) + return None + + f.close() + return build_config + + +# Generate build.ninja and objdiff.json +def generate_build(config): + build_config = load_build_config(config, config.out_path() / "config.json") + generate_build_ninja(config, build_config) + generate_objdiff_config(config, build_config) + + +# Generate build.ninja +def generate_build_ninja(config, build_config): + config.validate() + + out = io.StringIO() + n = ninja_syntax.Writer(out) + n.variable("ninja_required_version", "1.3") + n.newline() + + configure_script = os.path.relpath(os.path.abspath(sys.argv[0])) + python_lib = os.path.relpath(__file__) + python_lib_dir = os.path.dirname(python_lib) + n.comment("The arguments passed to configure.py, for rerunning it.") + n.variable("configure_args", sys.argv[1:]) + n.variable("python", f'"{sys.executable}"') + n.newline() + + ### + # Variables + ### + n.comment("Variables") + ldflags = " ".join(config.ldflags) + if config.generate_map: + ldflags += " -mapunused" + if config.debug: + ldflags += " -g" + n.variable("ldflags", ldflags) + n.variable("mw_version", config.linker_version) + n.newline() + + ### + # Tooling + ### + n.comment("Tooling") + + build_path = config.out_path() + build_tools_path = config.build_dir / "tools" + download_tool = config.tools_dir / "download_tool.py" + n.rule( + name="download_tool", + command=f"$python {download_tool} $tool $out --tag $tag", + description="TOOL $out", + ) + + if config.build_dtk_path: + dtk = build_tools_path / "release" / f"dtk{EXE}" + n.rule( + name="cargo", + command="cargo build --release --manifest-path $in --bin $bin --target-dir $target", + description="CARGO $bin", + depfile=path(Path("$target") / "release" / "$bin.d"), + deps="gcc", + ) + n.build( + outputs=path(dtk), + rule="cargo", + inputs=path(config.build_dtk_path / "Cargo.toml"), + implicit=path(config.build_dtk_path / "Cargo.lock"), + variables={ + "bin": "dtk", + "target": build_tools_path, + }, + ) + elif config.dtk_tag: + dtk = build_tools_path / f"dtk{EXE}" + n.build( + outputs=path(dtk), + rule="download_tool", + implicit=path(download_tool), + variables={ + "tool": "dtk", + "tag": config.dtk_tag, + }, + ) + else: + sys.exit("ProjectConfig.dtk_tag missing") + + if config.sjiswrap_path: + sjiswrap = config.sjiswrap_path + elif config.sjiswrap_tag: + sjiswrap = build_tools_path / "sjiswrap.exe" + n.build( + outputs=path(sjiswrap), + rule="download_tool", + implicit=path(download_tool), + variables={ + "tool": "sjiswrap", + "tag": config.sjiswrap_tag, + }, + ) + else: + sys.exit("ProjectConfig.sjiswrap_tag missing") + + # Only add an implicit dependency on wibo if we download it + wrapper = config.wrapper + wrapper_implicit = None + if ( + config.wibo_tag is not None + and sys.platform == "linux" + and platform.machine() in ("i386", "x86_64") + and config.wrapper is None + ): + wrapper = build_tools_path / "wibo" + wrapper_implicit = wrapper + n.build( + outputs=path(wrapper), + rule="download_tool", + implicit=path(download_tool), + variables={ + "tool": "wibo", + "tag": config.wibo_tag, + }, + ) + if not is_windows() and wrapper is None: + wrapper = "wine" + wrapper_cmd = f"{wrapper} " if wrapper else "" + + compilers_implicit = None + if config.compilers_path: + compilers = config.compilers_path + elif config.compilers_tag: + compilers = config.build_dir / "compilers" + compilers_implicit = compilers + n.build( + outputs=path(compilers), + rule="download_tool", + implicit=path(download_tool), + variables={ + "tool": "compilers", + "tag": config.compilers_tag, + }, + ) + + n.newline() + + ### + # Build rules + ### + compiler_path = compilers / "$mw_version" + + # MWCC + mwcc = compiler_path / "mwcceppc.exe" + mwcc_cmd = f"{wrapper_cmd}{mwcc} $cflags -MMD -c $in -o $basedir" + mwcc_implicit = [compilers_implicit or mwcc, wrapper_implicit] + + # MWCC with UTF-8 to Shift JIS wrapper + mwcc_sjis_cmd = f"{wrapper_cmd}{sjiswrap} {mwcc} $cflags -MMD -c $in -o $basedir" + mwcc_sjis_implicit = [*mwcc_implicit, sjiswrap] + + # MWLD + mwld = compiler_path / "mwldeppc.exe" + mwld_cmd = f"{wrapper_cmd}{mwld} $ldflags -o $out @$out.rsp" + mwld_implicit = [compilers_implicit or mwld, wrapper_implicit] + + if os.name != "nt": + transform_dep = config.tools_dir / "transform_dep.py" + mwcc_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" + mwcc_sjis_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" + mwcc_implicit.append(transform_dep) + mwcc_sjis_implicit.append(transform_dep) + + n.comment("Link ELF file") + n.rule( + name="link", + command=mwld_cmd, + description="LINK $out", + rspfile="$out.rsp", + rspfile_content="$in_newline", + ) + n.newline() + + n.comment("Generate DOL") + n.rule( + name="elf2dol", + command=f"{dtk} elf2dol $in $out", + description="DOL $out", + ) + n.newline() + + n.comment("Generate REL(s)") + makerel_rsp = build_path / "makerel.rsp" + n.rule( + name="makerel", + command=f"{dtk} rel make -w -c $config @{makerel_rsp}", + description="REL", + rspfile=path(makerel_rsp), + rspfile_content="$in_newline", + ) + n.newline() + + n.comment("MWCC build") + n.rule( + name="mwcc", + command=mwcc_cmd, + description="MWCC $out", + depfile="$basefile.d", + deps="gcc", + ) + n.newline() + + n.comment("MWCC build (with UTF-8 to Shift JIS wrapper)") + n.rule( + name="mwcc_sjis", + command=mwcc_sjis_cmd, + description="MWCC $out", + depfile="$basefile.d", + deps="gcc", + ) + n.newline() + + n.comment("Host build") + n.variable("host_cflags", "-I include -Wno-trigraphs") + n.variable( + "host_cppflags", + "-std=c++98 -I include -fno-exceptions -fno-rtti -D_CRT_SECURE_NO_WARNINGS -Wno-trigraphs -Wno-c++11-extensions", + ) + n.rule( + name="host_cc", + command="clang $host_cflags -c -o $out $in", + description="CC $out", + ) + n.rule( + name="host_cpp", + command="clang++ $host_cppflags -c -o $out $in", + description="CXX $out", + ) + n.newline() + + ### + # Source files + ### + n.comment("Source files") + build_src_path = build_path / "src" + build_host_path = build_path / "host" + build_config_path = build_path / "config.json" + + def map_path(path): + return path.parent / (path.name + ".MAP") + + class LinkStep: + def __init__(self, config): + self.name = config["name"] + self.module_id = config["module_id"] + self.ldscript = config["ldscript"] + self.entry = config["entry"] + self.inputs = [] + + def add(self, obj): + self.inputs.append(obj) + + def output(self): + if self.module_id == 0: + return build_path / f"{self.name}.dol" + else: + return build_path / self.name / f"{self.name}.rel" + + def partial_output(self): + if self.module_id == 0: + return build_path / f"{self.name}.elf" + else: + return build_path / self.name / f"{self.name}.plf" + + def write(self, n): + n.comment(f"Link {self.name}") + if self.module_id == 0: + elf_path = build_path / f"{self.name}.elf" + dol_path = build_path / f"{self.name}.dol" + elf_ldflags = f"$ldflags -lcf {self.ldscript}" + if config.generate_map: + elf_map = map_path(elf_path) + elf_ldflags += f" -map {elf_map}" + else: + elf_map = None + n.build( + outputs=path(elf_path), + rule="link", + inputs=path(self.inputs), + implicit=path([self.ldscript, *mwld_implicit]), + implicit_outputs=path(elf_map), + variables={"ldflags": elf_ldflags}, + ) + n.build( + outputs=path(dol_path), + rule="elf2dol", + inputs=path(elf_path), + implicit=path(dtk), + ) + else: + preplf_path = build_path / self.name / f"{self.name}.preplf" + plf_path = build_path / self.name / f"{self.name}.plf" + preplf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r" + plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -m {self.entry} -r1 -strip_partial -lcf {self.ldscript}" + if config.generate_map: + preplf_map = map_path(preplf_path) + preplf_ldflags += f" -map {preplf_map}" + plf_map = map_path(plf_path) + plf_ldflags += f" -map {plf_map}" + else: + preplf_map = None + plf_map = None + n.build( + outputs=path(preplf_path), + rule="link", + inputs=path(self.inputs), + implicit=path(mwld_implicit), + implicit_outputs=path(preplf_map), + variables={"ldflags": preplf_ldflags}, + ) + n.build( + outputs=path(plf_path), + rule="link", + inputs=path(self.inputs), + implicit=path([self.ldscript, preplf_path, *mwld_implicit]), + implicit_outputs=path(plf_map), + variables={"ldflags": plf_ldflags}, + ) + n.newline() + + if build_config: + link_steps = [] + used_compiler_versions = set() + source_inputs = [] + host_source_inputs = [] + source_added = set() + + def add_unit(build_obj, link_step): + obj_path, obj_name = build_obj["object"], build_obj["name"] + result = config.find_object(obj_name) + if not result: + if config.warn_missing_config and not build_obj["autogenerated"]: + print(f"Missing configuration for {obj_name}") + link_step.add(obj_path) + return + + lib, obj = result + lib_name = lib["lib"] + + options = obj.options + completed = obj.completed + + unit_src_path = config.src_dir / options["source"] + if not unit_src_path.exists(): + if config.warn_missing_source: + print(f"Missing source file {unit_src_path}") + link_step.add(obj_path) + return + + mw_version = options["mw_version"] or lib["mw_version"] + cflags = options["cflags"] or lib["cflags"] + if type(cflags) is list: + cflags_str = " ".join(cflags) + else: + cflags_str = str(cflags) + used_compiler_versions.add(mw_version) + + base_object = Path(obj.name).with_suffix("") + src_obj_path = build_src_path / f"{base_object}.o" + src_base_path = build_src_path / base_object + + if src_obj_path not in source_added: + source_added.add(src_obj_path) + + n.comment(f"{obj_name}: {lib_name} (linked {completed})") + n.build( + outputs=path(src_obj_path), + rule="mwcc_sjis" if options["shiftjis"] else "mwcc", + inputs=path(unit_src_path), + variables={ + "mw_version": path(Path(mw_version)), + "cflags": cflags_str, + "basedir": os.path.dirname(src_base_path), + "basefile": path(src_base_path), + }, + implicit=path( + mwcc_sjis_implicit if options["shiftjis"] else mwcc_implicit + ), + ) + + if lib["host"]: + host_obj_path = build_host_path / f"{base_object}.o" + host_base_path = build_host_path / base_object + n.build( + outputs=path(host_obj_path), + rule="host_cc" if unit_src_path.suffix == ".c" else "host_cpp", + inputs=path(unit_src_path), + variables={ + "basedir": os.path.dirname(host_base_path), + "basefile": path(host_base_path), + }, + ) + if options["add_to_all"]: + host_source_inputs.append(host_obj_path) + n.newline() + + if options["add_to_all"]: + source_inputs.append(src_obj_path) + + if completed: + obj_path = src_obj_path + link_step.add(obj_path) + + # Add DOL link step + link_step = LinkStep(build_config) + for unit in build_config["units"]: + add_unit(unit, link_step) + link_steps.append(link_step) + + if config.build_rels: + # Add REL link steps + for module in build_config["modules"]: + module_link_step = LinkStep(module) + for unit in module["units"]: + add_unit(unit, module_link_step) + link_steps.append(module_link_step) + n.newline() + + # Check if all compiler versions exist + for mw_version in used_compiler_versions: + mw_path = compilers / mw_version / "mwcceppc.exe" + if config.compilers_path and not os.path.exists(mw_path): + sys.exit(f"Compiler {mw_path} does not exist") + + # Check if linker exists + mw_path = compilers / config.linker_version / "mwldeppc.exe" + if config.compilers_path and not os.path.exists(mw_path): + sys.exit(f"Linker {mw_path} does not exist") + + ### + # Link + ### + for step in link_steps: + step.write(n) + n.newline() + + ### + # Generate RELs + ### + rel_outputs = list( + map( + lambda step: step.output(), + filter(lambda step: step.module_id != 0, link_steps), + ) + ) + if len(rel_outputs) > 0: + n.comment("Generate RELs") + n.build( + outputs=path(rel_outputs), + rule="makerel", + inputs=path(list(map(lambda step: step.partial_output(), link_steps))), + implicit=path([dtk, config.config_path]), + variables={"config": path(config.config_path)}, + ) + n.newline() + + ### + # Helper rule for building all source files + ### + n.comment("Build all source files") + n.build( + outputs="all_source", + rule="phony", + inputs=path(source_inputs), + ) + n.newline() + + ### + # Helper rule for building all source files, with a host compiler + ### + n.comment("Build all source files with a host compiler") + n.build( + outputs="all_source_host", + rule="phony", + inputs=path(host_source_inputs), + ) + n.newline() + + ### + # Check hash + ### + n.comment("Check hash") + ok_path = build_path / "ok" + n.rule( + name="check", + command=f"{dtk} shasum -q -c $in -o $out", + description="CHECK $in", + ) + n.build( + outputs=path(ok_path), + rule="check", + inputs=path(config.check_sha_path), + implicit=path([dtk, *map(lambda step: step.output(), link_steps)]), + ) + n.newline() + + ### + # Calculate progress + ### + n.comment("Calculate progress") + progress_path = build_path / "progress.json" + n.rule( + name="progress", + command=f"$python {configure_script} $configure_args progress", + description="PROGRESS", + ) + n.build( + outputs=path(progress_path), + rule="progress", + implicit=path([ok_path, configure_script, python_lib, config.config_path]), + ) + + ### + # Helper tools + ### + # TODO: make these rules work for RELs too + dol_link_step = link_steps[0] + dol_elf_path = dol_link_step.partial_output() + n.comment("Check for mismatching symbols") + n.rule( + name="dol_diff", + command=f"{dtk} -L error dol diff $in", + description=f"DIFF {dol_elf_path}", + ) + n.build( + inputs=path([config.config_path, dol_elf_path]), + outputs="dol_diff", + rule="dol_diff", + ) + n.build( + outputs="diff", + rule="phony", + inputs="dol_diff", + ) + n.newline() + + n.comment("Apply symbols from linked ELF") + n.rule( + name="dol_apply", + command=f"{dtk} dol apply $in", + description=f"APPLY {dol_elf_path}", + ) + n.build( + inputs=path([config.config_path, dol_elf_path]), + outputs="dol_apply", + rule="dol_apply", + implicit=path([ok_path]), + ) + n.build( + outputs="apply", + rule="phony", + inputs="dol_apply", + ) + n.newline() + + ### + # Split DOL + ### + n.comment("Split DOL into relocatable objects") + n.rule( + name="split", + command=f"{dtk} dol split $in $out_dir", + description="SPLIT $in", + depfile="$out_dir/dep", + deps="gcc", + ) + n.build( + inputs=path(config.config_path), + outputs=path(build_config_path), + rule="split", + implicit=path(dtk), + variables={"out_dir": path(build_path)}, + ) + n.newline() + + ### + # Regenerate on change + ### + n.comment("Reconfigure on change") + n.rule( + name="configure", + command=f"$python {configure_script} $configure_args", + generator=True, + description=f"RUN {configure_script}", + ) + n.build( + outputs="build.ninja", + rule="configure", + implicit=path( + [ + build_config_path, + configure_script, + python_lib, + Path(python_lib_dir) / "ninja_syntax.py", + ] + ), + ) + n.newline() + + ### + # Default rule + ### + n.comment("Default rule") + if build_config: + n.default(path(progress_path)) + else: + n.default(path(build_config_path)) + + # Write build.ninja + with open("build.ninja", "w", encoding="utf-8") as f: + f.write(out.getvalue()) + out.close() + + +# Generate objdiff.json +def generate_objdiff_config(config, build_config): + if not build_config: + return + + objdiff_config = { + "min_version": "0.4.3", + "custom_make": "ninja", + "build_target": False, + "watch_patterns": [ + "*.c", + "*.cp", + "*.cpp", + "*.h", + "*.hpp", + "*.py", + "*.yml", + "*.txt", + "*.json", + ], + "units": [], + } + + build_path = config.out_path() + + def add_unit(build_obj, module_name): + if build_obj["autogenerated"]: + # Skip autogenerated objects + return + + obj_path, obj_name = build_obj["object"], build_obj["name"] + base_object = Path(obj_name).with_suffix("") + unit_config = { + "name": unix_str(Path(module_name) / base_object), + "target_path": unix_str(obj_path), + } + + result = config.find_object(obj_name) + if not result: + objdiff_config["units"].append(unit_config) + return + + lib, obj = result + unit_src_path = config.src_dir / obj.options["source"] + if not unit_src_path.exists(): + objdiff_config["units"].append(unit_config) + return + + cflags = obj.options["cflags"] or lib["cflags"] + src_obj_path = build_path / "src" / f"{base_object}.o" + + reverse_fn_order = False + if type(cflags) is list: + for flag in cflags: + if not flag.startswith("-inline "): + continue + for value in flag.split(" ")[1].split(","): + if value == "deferred": + reverse_fn_order = True + elif value == "nodeferred": + reverse_fn_order = False + + unit_config["base_path"] = unix_str(src_obj_path) + unit_config["reverse_fn_order"] = reverse_fn_order + unit_config["complete"] = obj.completed + objdiff_config["units"].append(unit_config) + + # Add DOL units + for unit in build_config["units"]: + add_unit(unit, build_config["name"]) + + # Add REL units + for module in build_config["modules"]: + for unit in module["units"]: + add_unit(unit, module["name"]) + + # Write objdiff.json + with open("objdiff.json", "w", encoding="utf-8") as w: + json.dump(objdiff_config, w, indent=4) + + +# Calculate, print and write progress to progress.json +def calculate_progress(config): + out_path = config.out_path() + build_config = load_build_config(config, out_path / "config.json") + if not build_config: + return + + class ProgressUnit: + def __init__(self, name): + self.name = name + self.code_total = 0 + self.code_progress = 0 + self.data_total = 0 + self.data_progress = 0 + self.objects_progress = 0 + self.objects_total = 0 + self.objects = set() + + def add(self, build_obj): + self.code_total += build_obj["code_size"] + self.data_total += build_obj["data_size"] + + # Avoid counting the same object in different modules twice + include_object = build_obj["name"] not in self.objects + if include_object: + self.objects.add(build_obj["name"]) + self.objects_total += 1 + + if build_obj["autogenerated"]: + # Skip autogenerated objects + return + + result = config.find_object(build_obj["name"]) + if not result: + return + + _, obj = result + if not obj.completed: + return + + self.code_progress += build_obj["code_size"] + self.data_progress += build_obj["data_size"] + if include_object: + self.objects_progress += 1 + + def code_frac(self): + return self.code_progress / self.code_total + + def data_frac(self): + return self.data_progress / self.data_total + + # Add DOL units + all_progress = ProgressUnit("All") if config.progress_all else None + dol_progress = ProgressUnit("DOL") + for unit in build_config["units"]: + if all_progress: + all_progress.add(unit) + dol_progress.add(unit) + + # Add REL units + rels_progress = ProgressUnit("Modules") if config.progress_modules else None + modules_progress = [] + for module in build_config["modules"]: + progress = ProgressUnit(module["name"]) + modules_progress.append(progress) + for unit in module["units"]: + if all_progress: + all_progress.add(unit) + if rels_progress: + rels_progress.add(unit) + progress.add(unit) + + # Print human-readable progress + print("Progress:") + + def print_category(unit): + code_frac = unit.code_frac() + data_frac = unit.data_frac() + print( + f" {unit.name}: {code_frac:.2%} code, {data_frac:.2%} data ({unit.objects_progress} / {unit.objects_total} files)" + ) + print(f" Code: {unit.code_progress} / {unit.code_total} bytes") + print(f" Data: {unit.data_progress} / {unit.data_total} bytes") + + if all_progress: + print_category(all_progress) + print_category(dol_progress) + module_count = len(build_config["modules"]) + if module_count > 0: + print_category(rels_progress) + if config.progress_each_module: + for progress in modules_progress: + print_category(progress) + + # Generate and write progress.json + progress_json = {} + + def add_category(category, unit): + progress_json[category] = { + "code": unit.code_progress, + "code/total": unit.code_total, + "data": unit.data_progress, + "data/total": unit.data_total, + } + + if all_progress: + add_category("all", all_progress) + add_category("dol", dol_progress) + if len(build_config["modules"]) > 0: + if rels_progress: + add_category("modules", rels_progress) + if config.progress_each_module: + for progress in modules_progress: + add_category(progress.name, progress) + with open(out_path / "progress.json", "w", encoding="utf-8") as w: + json.dump(progress_json, w, indent=4) \ No newline at end of file diff --git a/tools/transform-dep.py b/tools/transform-dep.py deleted file mode 100644 index 5807ab1f..00000000 --- a/tools/transform-dep.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -# borrowed from prime-decomp -import argparse -import os -from platform import uname -from typing import List - -if os.name != 'nt': - wineprefix = os.environ.get('WINEPREFIX', os.path.join(os.environ['HOME'], '.wine')) - winedevices = os.path.join(wineprefix, 'dosdevices') - - -def in_wsl() -> bool: - # wsl1 has Microsoft, wsl2 has microsoft-standard - release = uname().release - return 'microsoft-standard' in release or 'Microsoft' in release - - -def convert_path(path: str) -> str: - # lowercase drive letter - path = path[0].lower() + path[1:] - if os.name == 'nt': - return path.replace('\\', '/') - elif path[0] == 'z': - # shortcut for z: - return path[2:].replace('\\', '/') - elif in_wsl(): - if path.startswith(r'\\wsl'): - # first part could be wsl$ or wsl.localhost - pos = path.find('\\', 2) - pos = path.find('\\', pos + 1) - path = path[pos:] - return path.replace('\\', '/') - else: - path = path[0:1] + path[2:] - return os.path.join('/mnt', path.replace('\\', '/')) - else: - # use $WINEPREFIX/dosdevices to resolve path - return os.path.realpath(os.path.join(winedevices, path.replace('\\', '/'))) - - -def import_d_file(in_file: str) -> str: - out_lines: List[str] = [] - - with open(in_file, 'r') as file: - it = iter(file) - line = next(it) - if line.endswith(' \\\n'): - out_lines.append(line[:-3].replace('\\', '/') + " \\\n") - else: - out_lines.append(line.replace('\\', '/')) - - return ''.join(out_lines) - - -def main(): - parser = argparse.ArgumentParser( - description="""Transform a .d file from Wine paths to normal paths""" - ) - parser.add_argument( - "d_file", - help="""Dependency file in""", - ) - parser.add_argument( - "d_file_out", - help="""Dependency file out""", - ) - args = parser.parse_args() - - output = import_d_file(args.d_file) - - with open(args.d_file_out, "w", encoding="UTF-8") as f: - f.write(output) - - -if __name__ == "__main__": - main() diff --git a/tools/transform_dep.py b/tools/transform_dep.py new file mode 100644 index 00000000..86bd2ecb --- /dev/null +++ b/tools/transform_dep.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 + +### +# Transforms .d files, converting Windows paths to Unix paths. +# Allows usage of the mwcc -MMD flag on platforms other than Windows. +# +# Usage: +# python3 tools/transform_dep.py build/src/file.d build/src/file.d +# +# If changes are made, please submit a PR to +# https://github.com/encounter/dtk-template +### + +import argparse +import os +from platform import uname + +wineprefix = os.path.join(os.environ["HOME"], ".wine") +if "WINEPREFIX" in os.environ: + wineprefix = os.environ["WINEPREFIX"] +winedevices = os.path.join(wineprefix, "dosdevices") + + +def in_wsl() -> bool: + return "microsoft-standard" in uname().release + + +def import_d_file(in_file) -> str: + out_text = "" + + with open(in_file) as file: + for idx, line in enumerate(file): + if idx == 0: + if line.endswith(" \\\n"): + out_text += line[:-3].replace("\\", "/") + " \\\n" + else: + out_text += line.replace("\\", "/") + else: + suffix = "" + if line.endswith(" \\\n"): + suffix = " \\" + path = line.lstrip()[:-3] + else: + path = line.strip() + # lowercase drive letter + path = path[0].lower() + path[1:] + if path[0] == "z": + # shortcut for z: + path = path[2:].replace("\\", "/") + elif in_wsl(): + path = path[0:1] + path[2:] + path = os.path.join("/mnt", path.replace("\\", "/")) + else: + # use $WINEPREFIX/dosdevices to resolve path + path = os.path.realpath( + os.path.join(winedevices, path.replace("\\", "/")) + ) + out_text += "\t" + path + suffix + "\n" + + return out_text + + +def main(): + parser = argparse.ArgumentParser( + description="""Transform a .d file from Wine paths to normal paths""" + ) + parser.add_argument( + "d_file", + help="""Dependency file in""", + ) + parser.add_argument( + "d_file_out", + help="""Dependency file out""", + ) + args = parser.parse_args() + + output = import_d_file(args.d_file) + + with open(args.d_file_out, "w", encoding="UTF-8") as f: + f.write(output) + + +if __name__ == "__main__": + main() \ No newline at end of file