#!/usr/bin/env python3 import glob import argparse import sys import os import time import subprocess import re import tempfile import platform from shutil import which from ninja import BIN_DIR # local copy as the pip version doesn't have dyndeps in build() func import ninja_syntax os.environ['WINEDEBUG'] = '-all' os.environ['TMPDIR'] = tempfile.gettempdir() has_wine = bool(which('wine')) has_wibo = bool(which('wibo')) has_cpp = bool(which('cpp')) # Native preprocesor doesn't work under WSL if "microsoft-standard" in platform.uname().release: has_cpp = False # TODO: make r3000.h and asm.h case sensitive symlinks on linux def parse_arguments(): parser = argparse.ArgumentParser(description='MGS Ninja build script generator') # Optional parser.add_argument('--psyq_path', type=str, default=os.environ.get("PSYQ_SDK") or "../../psyq_sdk", help='Path to the root of the cloned PSYQ repo') parser.add_argument('--variant', type=str, default='main_exe', choices=['main_exe', 'vr_exe'], help='Variant to build: main_exe for MGS Integral Disc 1/2 (SLPM_862.47/SLPM_862.48), vr_exe for MGS Integral VR Disc (SLPM_862.49)') args = parser.parse_args() args.psyq_path = os.path.relpath(args.psyq_path).replace("\\","/") args.obj_directory = 'obj' if args.variant == 'main_exe' else 'obj_vr' args.defines = ['VR_EXE'] if args.variant == 'vr_exe' else [] print("psyq_path = " + args.psyq_path) return args def prefix(pfx, cmd): if pfx == "wibo" and has_wibo: return f"wibo {cmd}" if has_wine: return f"wine {cmd}" return cmd def ninja_run(): ninja = os.path.join(BIN_DIR, 'ninja') ninja_args = [] # TODO: pass through args to ninja? # warrnings that were probably in the original code # TODO: hide these when building locally warning_whitelist = [ r'sd_drv\.c:\d+: warning: `temp\' might be used uninitialized in this function', r'stream\.c:\d+: warning: `dir_idx\' might be used uninitialized in this function', r'sd_main\.c:\d+: warning: unused variable `buffer\'', r'sd_drv\.c:\d+: warning: unused variable `temp\'', r'radiomes\.c:\d+: warning: unused variable `pad\'', r'radiomem\.c:\d+: warning: unused variable `pad\'', r'item\.c:\d+: warning: `state\' might be used uninitialized in this function', r'memcard\.c:\d+: warning: `op\' might be used uninitialized in this function', r'memcard\.c:\d+: warning: `count\' might be used uninitialized in this function', r'door\.c:\d+: warning: unused variable `pad\'', r'motion\.c:\d+: warning: `time\' might be used uninitialized in this function', r'motion\.c:\d+: warning: unused variable `unused\'', r'motion\.c:\d+: warning: `pArchive2\' might be used uninitialized in this function', r'motion\.c:\d+: warning: `shift2\' might be used uninitialized in this function', r'motion\.c:\d+: warning: `archive\' might be used uninitialized in this function', r'memcard\.c:\d+: warning: `return\' with no value, in function returning non-void', r'main\.c:\d+: warning: control reaches end of non-void function', r'control\.c:\d+: warning: `vy\' might be used uninitialized in this function', r'font\.c:\d+: warning: `m2\' might be used uninitialized in this function', r'radar\.c:\d+: warning: `pWalls\' might be used uninitialized in this function', r'radar\.c:\d+: warning: `ppWalls\' might be used uninitialized in this function', r'sndtst\.c:\d+: warning: `pName\' might be used uninitialized in this function', r'sndtst\.c:\d+: warning: `code\' might be used uninitialized in this function', r'select\.c:\d+: warning: `gcl_int\' might be used uninitialized in this function', r'select\.c:\d+: warning: `gcl_string\' might be used uninitialized in this function', r'mts_new\.c:\d+: warning: control reaches end of non-void function', r'overlay_bss\.c:\d+: warning: `s00a_dword_800E1120\' defined but not used', r'mosaic\.c:\d+: warning: unused variable `unused\'', r'vib_edit.c:\d+: warning: too many arguments for format', r'action.c:\d+: warning: assignment of read-only location', r'sphere.c:\d+: warning: `xoff\' might be used uninitialized in this function', r'sphere.c:\d+: warning: `yoff\' might be used uninitialized in this function', r'sphere.c:\d+: warning: `tpage\' might be used uninitialized in this function', r'sphere.c:\d+: warning: `clut\' might be used uninitialized in this function', r'sphere.c:\d+: warning: `u0\' might be used uninitialized in this function', r'sphere.c:\d+: warning: `v0\' might be used uninitialized in this function', r'sphere.c:\d+: warning: `u1\' might be used uninitialized in this function', r'sphere.c:\d+: warning: `v1\' might be used uninitialized in this function', ] if os.environ.get('APPVEYOR'): with subprocess.Popen([ninja] + ninja_args, stdout=subprocess.PIPE, encoding='utf8') as proc: for line in proc.stdout: sys.stdout.write(line) if 'warning: ' in line: found = False for whitelisted in warning_whitelist: if re.search(whitelisted, line): found = True break if not found: print('^ this warning must be fixed in order to merge', file=sys.stderr) print('.. and all other warnings minus these:') for whitelisted in warning_whitelist: print(' ', whitelisted) sys.exit(1) return proc.wait() else: return subprocess.call([ninja] + ninja_args) args = parse_arguments() f = open("build.ninja", "w") ninja = ninja_syntax.Writer(f) ninja.variable("psyq_path", args.psyq_path) ninja.newline() ninja.variable("psyq_path_backslashed", args.psyq_path.replace('/', '\\')) ninja.newline() ninja.variable("psyq_asmpsx_44_exe", prefix("wibo", "$psyq_path/psyq_4.4/bin/asmpsx.exe")) ninja.newline() preprocessor_defines = ' '.join(f'-D{define}' for define in args.defines) if has_cpp: ninja.variable("psyq_c_preprocessor_44_exe", f"cpp -nostdinc {preprocessor_defines}") else: ninja.variable("psyq_c_preprocessor_44_exe", prefix("wine", f"$psyq_path/psyq_4.4/bin/CPPPSX.exe {preprocessor_defines}")) ninja.newline() ninja.variable("psyq_cc_44_exe", prefix("wine", "$psyq_path/psyq_4.4/bin/CC1PSX.EXE")) ninja.newline() ninja.variable("psyq_aspsx_44_exe", prefix("wibo", "$psyq_path/psyq_4.4/bin/aspsx.exe")) ninja.newline() if has_cpp: ninja.variable("psyq_c_preprocessor_43_exe", f"cpp -nostdinc {preprocessor_defines}") else: ninja.variable("psyq_c_preprocessor_43_exe", prefix("wine", f"$psyq_path/psyq_4.3/bin/CPPPSX.exe {preprocessor_defines}")) ninja.newline() ninja.variable("psyq_cc_43_exe", prefix("wine", "$psyq_path/psyq_4.3/bin/CC1PSX.EXE")) ninja.newline() ninja.variable("psyq_aspsx_2_56_exe", prefix("wibo", "$psyq_path/ASPSX/2.56/ASPSX.EXE")) ninja.variable("psyq_aspsx_2_81_exe", prefix("wibo", "$psyq_path/ASPSX/2.81/ASPSX.EXE")) ninja.variable("psyq_psylink_exe", prefix("wibo", "$psyq_path/psyq_4.4/bin/psylink.exe")) ninja.newline() ninja.variable("psyq_psylink_overlay_fopen_mod_exe", prefix("wibo", "$psyq_path/psyq_4.4/bin/psylink_overlay_fopen_mod.exe")) ninja.newline() ninja.variable("src_dir", "../src") ninja.newline() # /l = produce linkable output file # /q = run quietly ninja.rule("psyq_asmpsx_assemble", "$psyq_asmpsx_44_exe /l /q $in,$out", "Assemble $in -> $out") ninja.newline() includes = "-I " + args.psyq_path + "/psyq_4.4/INCLUDE" + " -I $src_dir" ninja.rule("psyq_c_preprocess_44", "$psyq_c_preprocessor_44_exe -undef -D__GNUC__=2 -D__OPTIMIZE__ " + includes + " -lang-c -Dmips -D__mips__ -D__mips -Dpsx -D__psx__ -D__psx -D_PSYQ -D__EXTENSIONS__ -D_MIPSEL -D__CHAR_UNSIGNED__ -D_LANGUAGE_C -DLANGUAGE_C $in $out", "Preprocess $in -> $out") ninja.newline() ninja.rule("convert_c_encoding", f"{sys.executable} $src_dir/../build/convjp.py $in $out", "Convert $in from UTF-8 to EUC-JP") ninja.newline() # generate header deps, adds edges to the build graph for the next build -M option will write header deps ninja.rule("psyq_c_preprocess_44_headers", "$psyq_c_preprocessor_44_exe -M -undef -D__GNUC__=2 -D__OPTIMIZE__ " + includes + " -lang-c -Dmips -D__mips__ -D__mips -Dpsx -D__psx__ -D__psx -D_PSYQ -D__EXTENSIONS__ -D_MIPSEL -D__CHAR_UNSIGNED__ -D_LANGUAGE_C -DLANGUAGE_C $in $out", "Preprocess for includes $in -> $out") ninja.newline() ninja.rule("header_deps", f"{sys.executable} $src_dir/../build/hash_include_msvc_formatter.py $in $out", "Include deps fix $in -> $out", deps="msvc") ninja.newline() ninja.rule("asm_include_preprocess_44", f"{sys.executable} $src_dir/../build/include_asm_preprocess.py $in $out", "Include asm preprocess $in -> $out") ninja.newline() ninja.rule("asm_include_postprocess", f"{sys.executable} $src_dir/../build/include_asm_fixup.py $in $out", "Include asm postprocess $in -> $out") ninja.newline() ninja.variable("gSize", "8") ninja.newline() ninja.rule("psyq_cc_44", "$psyq_cc_44_exe -quiet -O2 -G $gSize -g -Wall $in -o $out""", "Compile $in -> $out") ninja.newline() ninja.rule("psyq_aspsx_assemble_44_overlays", "$psyq_aspsx_44_exe -q -G0 -s-overlay $in -o $out""", "Compile $in -> $out") ninja.newline() ninja.rule("psyq_aspsx_assemble_44", "$psyq_aspsx_44_exe -q $in -o $out", "Assemble $in -> $out") ninja.newline() ninja.rule("psyq_aspsx_assemble_2_81_overlays", "$psyq_aspsx_2_81_exe -q -G0 -s-overlay $in -o $out""", "Compile $in -> $out") ninja.newline() ninja.rule("psyq_aspsx_assemble_2_81", "$psyq_aspsx_2_81_exe -q $in -o $out", "Assemble $in -> $out") ninja.newline() # For some reason 4.3 cc needs TMPDIR set to something that exists else it will just die with "CC1PSX.exe: /cta04280: No such file or directory" ninja.rule("psyq_cc_43", "$psyq_cc_43_exe -quiet -O2 -G $gSize -g -Wall $in -o $out", "Compile $in -> $out") ninja.newline() ninja.rule("psyq_aspsx_assemble_2_56", "$psyq_aspsx_2_56_exe -q $in -o $out", "Assemble $in -> $out") ninja.newline() ninja.rule("linker_command_file_preprocess", f"{sys.executable} $src_dir/../build/linker_command_file_preprocess.py $in $psyq_sdk $out {' '.join(args.defines)} $overlay $overlay_suffix", "Preprocess $in -> $out") ninja.newline() # For some reason VR executable links with PsyQ 4.5!? psqy_lib = f'{args.psyq_path}/psyq_4.5/LIB' if args.variant == 'vr_exe' else f'{args.psyq_path}/psyq_4.4/LIB' ninja.rule("psylink", f"$psyq_psylink_exe /l {psqy_lib} /c /n 4000 /q /gp .sdata /m \"@$src_dir/../{args.obj_directory}/linker_command_file$suffix.txt\",$src_dir/../{args.obj_directory}/_mgsi$suffix.cpe,$src_dir/../{args.obj_directory}/asm$suffix.sym,$src_dir/../{args.obj_directory}/asm$suffix.map", "Link $out") ninja.newline() ninja.rule("create_dummy_file_overlays", f"{sys.executable} $src_dir/../build/create_dummy_file.py $src_dir/../{args.obj_directory}/$overlay_bin $src_dir/../{args.obj_directory}/$overlay_bss_bin", "Create dummy files $overlay_bin, $overlay_bss_bin") ninja.newline() ninja.rule("psylink_overlay_fopen_mod", f"$psyq_psylink_overlay_fopen_mod_exe /l {psqy_lib} /c /n 4000 /q /gp .sdata /m \"@$src_dir/../{args.obj_directory}/linker_command_file$suffix.txt\",$src_dir/../{args.obj_directory}/_mgsi$suffix.cpe,$src_dir/../{args.obj_directory}/asm$suffix.sym,$src_dir/../{args.obj_directory}/asm$suffix.map", "Link (uninitialized) $out") ninja.newline() ninja.rule("uninitializer", f"{sys.executable} $src_dir/../build/uninitializer.py inject $in $out", "Uninitializer $in -> $out") ninja.newline() # TODO: update the tool so we can set the output name optionally # cmd /c doesn't like forward slashed relative paths ninja.rule("cpe2exe", prefix("wine", "cmd /c \"$psyq_path_backslashed\\psyq_4.3\\bin\\cpe2exe.exe -CJ $in > NUL\""), "cpe2exe $in -> $out") ninja.newline() ninja.rule("hash_check", f"{sys.executable} $src_dir/../build/compare.py $in", "Hash check $in") ninja.newline() def create_psyq_ini(sdkDir, psyqDir): data = "" with open(sdkDir + "/" + psyqDir + "/bin/psyq.ini.template", 'r') as file: data = file.read() data = data.replace("$PSYQ_PATH", sdkDir + "/" + psyqDir) data = data.replace("/", "\\") ini_file = open(sdkDir + "/" + psyqDir + "/bin/psyq.ini", "w") ini_file.write(data) def init_psyq_ini_files(sdkDir): create_psyq_ini(sdkDir, "psyq_4.3") create_psyq_ini(sdkDir, "psyq_4.4") def get_files_recursive(path, ext): collectedFiles = [] # r=root, d=directories, f = files for r, d, f in os.walk(path): for file in f: if file.endswith(ext): collectedFiles.append(os.path.join(r, file)) return collectedFiles def get_file_global_size(file): if "overlays/" in file or "mts/" in file or "SD/" in file: return "0" g0_list = [ "/Equip/", "/Bullet/", "/Thing/", "/Okajima/", "Game/item.c", # todo figure out if correct, why not all .c files in this dir ?? "anime.c", # ditto "vibrate.c", "/Takabe/", "/libfs/", "Kojo/demo.c", "Kojo/demothrd.c", "strctrl.c", "jimctrl.c", "memcard.c", "dgd.c", "sna_hzd.c", ] if any(i in file for i in g0_list): return "0" return "8" def gen_build_target(targetName): ninja.comment("Build target " + targetName) asmFiles = get_files_recursive("../asm", ".s") print("Got " + str(len(asmFiles)) + " asm files") cFiles = get_files_recursive("../src", ".c") print("Got " + str(len(cFiles)) + " source files") linkerDeps = [] # TODO: Use the correct toolchain and -G flag for each c file # TODO: .h file deps of .c files # build .s files for asmFile in asmFiles: asmFile = asmFile.replace("\\", "/") asmOFile = asmFile.replace("/asm/", f"/{args.obj_directory}/") asmOFile = asmOFile.replace(".s", ".obj") #print("Build step " + asmFile + " -> " + asmOFile) ninja.build(asmOFile, "psyq_asmpsx_assemble", asmFile) linkerDeps.append(asmOFile) # build .c files for cFile in cFiles: cFile = cFile.replace("\\", "/") cOFile = cFile.replace("/src/", f"/{args.obj_directory}/") cPreProcHeadersFile = cOFile.replace(".c", ".c.preproc.headers") cPreProcHeadersFixedFile = cOFile.replace(".c", ".c.preproc.headers_fixed") cConvertedFile = cOFile.replace(".c", ".c.eucjp") cPreProcFile = cOFile.replace(".c", ".c.preproc") cDynDepFile = cOFile.replace(".c", ".c.dyndep") cAsmPreProcFile = cOFile.replace(".c", ".c.asm.preproc") cAsmPreProcFileDeps = cOFile.replace(".c", ".c.asm.preproc.deps") cAsmFile = cOFile.replace(".c", ".asm") cTempOFile = cOFile.replace(".c", "_fixme.obj") cOFile = cOFile.replace(".c", ".obj") #print("Build step " + asmFile + " -> " + asmOFile) ninja.build(cPreProcHeadersFile, "psyq_c_preprocess_44_headers", cFile) ninja.build(cPreProcHeadersFixedFile, "header_deps", cPreProcHeadersFile) compiler = "psyq_cc_44" if "mts/" in cFile or "SD/" in cFile: compiler = "psyq_cc_43" aspsx = "psyq_aspsx_assemble_44" if "overlays" in cFile: if args.variant == 'vr_exe': aspsx = "psyq_aspsx_assemble_2_81_overlays" else: aspsx = "psyq_aspsx_assemble_44_overlays" elif "mts/" in cFile or "SD/" in cFile: aspsx = "psyq_aspsx_assemble_2_56" elif args.variant == 'vr_exe': aspsx = "psyq_aspsx_assemble_2_81" ninja.build(cPreProcFile, "psyq_c_preprocess_44", cFile, implicit=[cPreProcHeadersFixedFile]) ninja.build([cAsmPreProcFile, cAsmPreProcFileDeps, cDynDepFile], "asm_include_preprocess_44", cPreProcFile) ninja.build(cConvertedFile, "convert_c_encoding", cAsmPreProcFile) ninja.build(cAsmFile, compiler, cConvertedFile, variables= { "gSize": get_file_global_size(cFile) }) ninja.build(cTempOFile, aspsx, cAsmFile) ninja.build(cOFile, "asm_include_postprocess", cTempOFile, implicit=[cAsmPreProcFileDeps, cDynDepFile], dyndep=cDynDepFile) linkerDeps.append(cOFile) # Build main exe # preprocess linker_command_file.txt linkerCommandFile = f"../{args.obj_directory}/linker_command_file.txt" ninja.build(linkerCommandFile, "linker_command_file_preprocess", f"linker_command_file.txt", variables={'psyq_sdk': args.psyq_path}) ninja.newline() # run the linker to generate the cpe cpeFile = f"../{args.obj_directory}/_mgsi.cpe" ninja.build(cpeFile, "psylink", implicit=linkerDeps + [linkerCommandFile]) ninja.newline() # cpe to exe exeFile = f"../{args.obj_directory}/_mgsi.exe" ninja.build(exeFile, "cpe2exe", cpeFile) ninja.newline() # Run linker separately for each overlay to make it possible # to share objects (same symbols) across overlays. OVERLAYS = [ "sound", "select1", "select2", "select3", "select4", "selectd", "change", "s16b", "camera", "select", "d11c", "s00a", "d03a", "s03e", "s03er", "title", "s01a", "d01a", "s02c", "s02e", "s16c", "s06a", "s02a", "s02d", "s12c", "s07a", "s04a", "s02b", ] if args.variant == 'vr_exe': OVERLAYS = [] for overlay in OVERLAYS: # It turns out that MGS overlays contain uninitialized memory # in: # - Padding between values in data/rdata (e.g. padding between strings) # - BSS # Our psylink doesn't have the same problem and it fills those # spaces correctly with 0s. # # To inject back the uninitialized memory we run the linker twice. # The first run uses unmodified psylink, so the generated overlay # has 0s in those described places. The second run uses a modified # version of psylink that writes on top of an existing file # (fopen(, "r+b") instead of fopen(, "wb")). Before the second run # we create a dummy file filled with a repeating non-zero byte - # this represents the uninitialized memory that we will detect. # After that we can use those two generated files, diff them # and combined with the uninitialized memory extracted from original # files to generate an overlay with uninitialized memory. # First run (LHS) linkerCommandFile = f"../{args.obj_directory}/linker_command_file_{overlay}_lhs.txt" linkerCommandPreprocessVars = { "overlay": f"OVERLAY={overlay}", "overlay_suffix": "OVERLAY_SUFFIX=lhs", "psyq_sdk": args.psyq_path } ninja.build(linkerCommandFile, "linker_command_file_preprocess", f"linker_command_file.txt", variables=linkerCommandPreprocessVars) ninja.newline() lhsOverlayFile = f"../{args.obj_directory}/{overlay}_lhs.bin" ninja.build(lhsOverlayFile, "psylink", implicit=linkerDeps + [linkerCommandFile], variables={"suffix": f"_{overlay}_lhs"}) ninja.newline() # Second run (RHS) linkerCommandFile = f"../{args.obj_directory}/linker_command_file_{overlay}_rhs.txt" linkerCommandPreprocessVars = { "overlay": f"OVERLAY={overlay}", "overlay_suffix": "OVERLAY_SUFFIX=rhs", "psyq_sdk": args.psyq_path } ninja.build(linkerCommandFile, "linker_command_file_preprocess", f"linker_command_file.txt", variables=linkerCommandPreprocessVars) ninja.newline() dummyFile = f"../{args.obj_directory}/{overlay}_rhs_bss.bin" ninja.build(dummyFile, "create_dummy_file_overlays", implicit=linkerDeps + [linkerCommandFile], variables={"overlay_bin": f"{overlay}_rhs.bin", "overlay_bss_bin": f"{overlay}_rhs_bss.bin"}) ninja.newline() rhsOverlayFile = f"../{args.obj_directory}/{overlay}_rhs.bin" ninja.build(rhsOverlayFile, "psylink_overlay_fopen_mod", implicit=linkerDeps + [linkerCommandFile, dummyFile], variables={"suffix": f"_{overlay}_rhs"}) ninja.newline() overlayFile = f"../{args.obj_directory}/{overlay}.bin" ninja.build(overlayFile, "uninitializer", inputs=[lhsOverlayFile, rhsOverlayFile, f"../um/{overlay}.bin"], variables={"overlay": f"{overlay}"}) ninja.newline() #init_psyq_ini_files(args.psyq_path) gen_build_target("SLPM_862.47") #gen_build_target("sound.bin") # TODO: all overlays f.close() time_before = time.time() exit_code = ninja_run() took = time.time() - time_before print(f'build took {took:.2f} seconds') if exit_code == 0: ret = subprocess.run([sys.executable, 'compare.py']) exit_code = ret.returncode if exit_code == 0: ret = subprocess.run([sys.executable, 'post_build_checkup.py']) exit_code = ret.returncode sys.exit(exit_code)