From 1c36d30a720efbd3b2c32a59295bf8be4a64b67a Mon Sep 17 00:00:00 2001 From: elijah-thomas774 Date: Fri, 22 Dec 2023 14:54:03 -0500 Subject: [PATCH] updated some project tools from the template --- configure.py | 4 +- tools/download_tool.py | 8 +-- tools/project.py | 121 ++++++++++++++++++++++++++++++++++------- 3 files changed, 104 insertions(+), 29 deletions(-) diff --git a/configure.py b/configure.py index cfa5fbe7..58b5fb48 100644 --- a/configure.py +++ b/configure.py @@ -119,7 +119,7 @@ if not is_windows(): config.wrapper = args.wrapper # Tool versions -config.compilers_tag = "1" +config.compilers_tag = "20231018" config.dtk_tag = "v0.6.5" config.sjiswrap_tag = "v1.1.1" config.wibo_tag = "0.6.9" @@ -380,4 +380,4 @@ elif args.mode == "progress": config.progress_each_module = args.verbose calculate_progress(config) else: - sys.exit("Unknown mode: " + args.mode) \ No newline at end of file + sys.exit("Unknown mode: " + args.mode) diff --git a/tools/download_tool.py b/tools/download_tool.py index 27099669..fef42d6e 100644 --- a/tools/download_tool.py +++ b/tools/download_tool.py @@ -16,7 +16,6 @@ import os import platform import shutil import stat -import sys import urllib.request import zipfile @@ -50,10 +49,7 @@ def wibo_url(tag): def compilers_url(tag): - if tag == "1": - return "https://cdn.discordapp.com/attachments/727918646525165659/1129759991696457728/GC_WII_COMPILERS.zip" - else: - sys.exit("Unknown compilers tag %s" % tag) + return f"https://files.decomp.dev/compilers_{tag}.zip" TOOLS = { @@ -90,4 +86,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tools/project.py b/tools/project.py index cdadec68..8ccfde1a 100644 --- a/tools/project.py +++ b/tools/project.py @@ -15,6 +15,7 @@ import json import os import platform import sys +import math from pathlib import Path from . import ninja_syntax @@ -56,13 +57,20 @@ class ProjectConfig: self.version = None # Version name self.warn_missing_config = False # Warn on missing unit configuration self.warn_missing_source = False # Warn on missing source file + self.rel_strip_partial = True # Generate PLFs with -strip_partial + self.rel_empty_file = None # Path to empty.c for generating empty RELs # Progress output and progress.json config self.progress_all = True # Include combined "all" category self.progress_modules = True # Include combined "modules" category - self.progress_each_module = ( - True # Include individual modules, disable for large numbers of modules - ) + self.progress_each_module = True # Include individual modules, disable for large numbers of modules + + # Progress fancy printing + self.progress_use_fancy = False + self.progress_code_fancy_frac = 0 + self.progress_code_fancy_item = "" + self.progress_data_fancy_frac = 0 + self.progress_data_fancy_item = "" def validate(self): required_attrs = [ @@ -98,6 +106,7 @@ class Object: self.options = { "add_to_all": True, "cflags": None, + "extra_cflags": None, "mw_version": None, "shiftjis": True, "source": name, @@ -461,7 +470,12 @@ def generate_build_ninja(config, build_config): preplf_path = build_path / self.name / f"{self.name}.preplf" plf_path = build_path / self.name / f"{self.name}.plf" preplf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r" - plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -m {self.entry} -r1 -strip_partial -lcf {self.ldscript}" + plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r1 -lcf {self.ldscript}" + if self.entry: + plf_ldflags += f" -m {self.entry}" + # -strip_partial is only valid with -m + if config.rel_strip_partial: + plf_ldflags += " -strip_partial" if config.generate_map: preplf_map = map_path(preplf_path) preplf_ldflags += f" -map {preplf_map}" @@ -495,6 +509,12 @@ def generate_build_ninja(config, build_config): host_source_inputs = [] source_added = set() + def make_cflags_str(cflags): + if isinstance(cflags, list): + return " ".join(cflags) + else: + return cflags + def add_unit(build_obj, link_step): obj_path, obj_name = build_obj["object"], build_obj["name"] result = config.find_object(obj_name) @@ -506,23 +526,24 @@ def generate_build_ninja(config, build_config): lib, obj = result lib_name = lib["lib"] + src_dir = Path(lib.get("src_dir", config.src_dir)) options = obj.options completed = obj.completed - unit_src_path = config.src_dir / options["source"] + unit_src_path = src_dir / options["source"] + if not unit_src_path.exists(): - if config.warn_missing_source: + if config.warn_missing_source or completed: print(f"Missing source file {unit_src_path}") link_step.add(obj_path) return mw_version = options["mw_version"] or lib["mw_version"] - cflags = options["cflags"] or lib["cflags"] - if type(cflags) is list: - cflags_str = " ".join(cflags) - else: - cflags_str = str(cflags) + cflags_str = make_cflags_str(options["cflags"] or lib["cflags"]) + if options["extra_cflags"] is not None: + extra_cflags_str = make_cflags_str(options["extra_cflags"]) + cflags_str += " " + extra_cflags_str used_compiler_versions.add(mw_version) base_object = Path(obj.name).with_suffix("") @@ -583,6 +604,18 @@ def generate_build_ninja(config, build_config): module_link_step = LinkStep(module) for unit in module["units"]: add_unit(unit, module_link_step) + # Add empty object to empty RELs + if len(module_link_step.inputs) == 0: + if not config.rel_empty_file: + sys.exit("ProjectConfig.rel_empty_file missing") + add_unit( + { + "object": None, + "name": config.rel_empty_file, + "autogenerated": True, + }, + module_link_step, + ) link_steps.append(module_link_step) n.newline() @@ -607,18 +640,41 @@ def generate_build_ninja(config, build_config): ### # Generate RELs ### - rel_outputs = list( - map( - lambda step: step.output(), - filter(lambda step: step.module_id != 0, link_steps), + generated_rels = [] + for link in build_config["links"]: + # Map module names to link steps + link_steps_local = list( + filter( + lambda step: step.name in link["modules"], + link_steps, + ) + ) + link_steps_local.sort(key=lambda step: step.module_id) + # RELs can be the output of multiple link steps, + # so we need to filter out duplicates + rels_to_generate = list( + filter( + lambda step: step.module_id != 0 + and not step.name in generated_rels, + link_steps_local, + ) + ) + if len(rels_to_generate) == 0: + continue + generated_rels.extend(map(lambda step: step.name, rels_to_generate)) + rel_outputs = list( + map( + lambda step: step.output(), + rels_to_generate, + ) ) - ) - if len(rel_outputs) > 0: n.comment("Generate RELs") n.build( outputs=path(rel_outputs), rule="makerel", - inputs=path(list(map(lambda step: step.partial_output(), link_steps))), + inputs=path( + list(map(lambda step: step.partial_output(), link_steps_local)) + ), implicit=path([dtk, config.config_path]), variables={"config": path(config.config_path)}, ) @@ -651,9 +707,10 @@ def generate_build_ninja(config, build_config): ### n.comment("Check hash") ok_path = build_path / "ok" + quiet = "-q " if len(link_steps) > 3 else "" n.rule( name="check", - command=f"{dtk} shasum -q -c $in -o $out", + command=f"{dtk} shasum {quiet} -c $in -o $out", description="CHECK $in", ) n.build( @@ -797,6 +854,7 @@ def generate_objdiff_config(config, build_config): "*.cpp", "*.h", "*.hpp", + "*.inc", "*.py", "*.yml", "*.txt", @@ -825,7 +883,10 @@ def generate_objdiff_config(config, build_config): return lib, obj = result - unit_src_path = config.src_dir / obj.options["source"] + src_dir = Path(lib.get("src_dir", config.src_dir)) + + unit_src_path = src_dir / obj.options["source"] + if not unit_src_path.exists(): objdiff_config["units"].append(unit_config) return @@ -874,8 +935,12 @@ def calculate_progress(config): def __init__(self, name): self.name = name self.code_total = 0 + self.code_fancy_frac = config.progress_code_fancy_frac + self.code_fancy_item = config.progress_code_fancy_item self.code_progress = 0 self.data_total = 0 + self.data_fancy_frac = config.progress_data_fancy_frac + self.data_fancy_item = config.progress_data_fancy_item self.data_progress = 0 self.objects_progress = 0 self.objects_total = 0 @@ -939,6 +1004,9 @@ def calculate_progress(config): print("Progress:") def print_category(unit): + if unit is None: + return + code_frac = unit.code_frac() data_frac = unit.data_frac() print( @@ -946,6 +1014,17 @@ def calculate_progress(config): ) print(f" Code: {unit.code_progress} / {unit.code_total} bytes") print(f" Data: {unit.data_progress} / {unit.data_total} bytes") + if config.progress_use_fancy: + print( + "\nYou have {} out of {} {} and collected {} out of {} {}.".format( + math.floor(code_frac * unit.code_fancy_frac), + unit.code_fancy_frac, + unit.code_fancy_item, + math.floor(data_frac * unit.data_fancy_frac), + unit.data_fancy_frac, + unit.data_fancy_item, + ) + ) if all_progress: print_category(all_progress) @@ -978,4 +1057,4 @@ def calculate_progress(config): for progress in modules_progress: add_category(progress.name, progress) with open(out_path / "progress.json", "w", encoding="utf-8") as w: - json.dump(progress_json, w, indent=4) \ No newline at end of file + json.dump(progress_json, w, indent=4)