mirror/tww
1
0
Fork 0
mirror of https://github.com/zeldaret/tww.git synced 2026-08-21 22:10:59 -04:00
Code Issues Packages Projects Releases Wiki Activity

Progress categories support

Browse Source
This commit is contained in:
Luke Street
2024-09-02 23:30:02 -06:00
parent 596fe9ab93
commit 5495d71f95
2 changed files with 423 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configure.py
+59 -8
View File
@@ -19,6 +19,7 @@ from typing import Any, Dict, List
from tools.project import (
Object,
ProgressCategory,
ProjectConfig,
calculate_progress,
generate_build,
@@ -144,6 +145,7 @@ if args.no_asm:
config.binutils_tag = "2.42-1"
config.compilers_tag = "20240706"
config.dtk_tag = "v0.9.2"
config.objdiff_tag = "v2.0.0-beta.4"
config.sjiswrap_tag = "v1.1.1"
config.wibo_tag = "0.6.11"
@@ -256,6 +258,7 @@ def DolphinLib(lib_name: str, objects: List[Object]) -> Dict[str, Any]:
"lib": lib_name,
"mw_version": "GC/1.2.5n",
"cflags": cflags_base,
"progress_category": "dolphin",
"host": False,
"objects": objects,
}
@@ -267,6 +270,7 @@ def Rel(lib_name: str, objects: List[Object]) -> Dict[str, Any]:
"lib": lib_name,
"mw_version": "GC/1.3.2",
"cflags": cflags_rel,
"progress_category": "game",
"host": True,
"objects": objects,
}
@@ -283,6 +287,7 @@ def JSystemLib(lib_name, objects):
"lib": lib_name,
"mw_version": "GC/1.3.2",
"cflags": cflags_framework,
"progress_category": "core",
"host": True,
"objects": objects,
}
@@ -293,14 +298,22 @@ Equivalent = config.non_matching # Object should be linked when configured with
config.warn_missing_config = True
config.warn_missing_source = False
config.progress_categories = [
ProgressCategory("all", "All"),
ProgressCategory("dol", "DOL"),
ProgressCategory("modules", "Modules"),
ProgressCategory("core", "Core"),
ProgressCategory("game", "Game"),
ProgressCategory("dolphin", "Dolphin"),
]
config.libs = [
{
"lib": "framework",
"lib": "machine",
"mw_version": "GC/1.3.2",
"cflags": cflags_framework,
"progress_category": "core",
"host": True,
"objects": [
# machine
Object(Matching, "m_Do/m_Do_main.cpp"),
Object(Matching, "m_Do/m_Do_printf.cpp"),
Object(Matching, "m_Do/m_Do_audio.cpp"),
@@ -318,11 +331,26 @@ config.libs = [
Object(NonMatching, "m_Do/m_Do_MemCardRWmng.cpp"),
Object(Matching, "m_Do/m_Do_gba_com.cpp"),
Object(Matching, "m_Do/m_Do_machine_exception.cpp"),
# c
],
},
{
"lib": "c",
"mw_version": "GC/1.3.2",
"cflags": cflags_framework,
"progress_category": "game",
"host": True,
"objects": [
Object(NonMatching, "c/c_damagereaction.cpp"),
Object(Matching, "c/c_dylink.cpp"),
],
},
{
"lib": "framework",
"mw_version": "GC/1.3.2",
"cflags": cflags_framework,
"progress_category": "core",
"host": True,
"objects": [
# f_ap
Object(Matching, "f_ap/f_ap_game.cpp"),
@@ -381,8 +409,15 @@ config.libs = [
Object(Matching, "f_pc/f_pc_draw.cpp"),
Object(Matching, "f_pc/f_pc_fstcreate_req.cpp"),
Object(Matching, "f_pc/f_pc_stdcreate_req.cpp"),
# dolzel
],
},
{
"lib": "dolzel",
"mw_version": "GC/1.3.2",
"cflags": cflags_framework,
"progress_category": "game",
"host": True,
"objects": [
Object(NonMatching, "d/d_stage.cpp"),
Object(NonMatching, "d/d_map.cpp"),
Object(Matching, "d/d_com_inf_game.cpp", extra_cflags=['-sym off']),
@@ -543,7 +578,15 @@ config.libs = [
Object(Matching, "d/d_wind_arrow.cpp"),
Object(NonMatching, "d/d_wpillar.cpp"),
Object(Matching, "d/d_wpot_water.cpp"),
],
},
{
"lib": "DynamicLink",
"mw_version": "GC/1.3.2",
"cflags": cflags_framework,
"progress_category": "core",
"host": True,
"objects": [
Object(Matching, "DynamicLink.cpp"),
],
},
@@ -551,6 +594,7 @@ config.libs = [
"lib": "SSystem",
"mw_version": "GC/1.3.2",
"cflags": cflags_framework,
"progress_category": "core",
"host": True,
"objects": [
Object(Matching, "SSystem/SComponent/c_malloc.cpp"),
@@ -1114,6 +1158,7 @@ config.libs = [
"lib": "Runtime.PPCEABI.H",
"mw_version": "GC/1.3",
"cflags": cflags_runtime,
"progress_category": "dolphin",
"host": False,
"objects": [
Object(Matching, "PowerPC_EABI_Support/Runtime/Src/__mem.c"),
@@ -1132,6 +1177,7 @@ config.libs = [
"lib": "MSL_C",
"mw_version": "GC/1.3",
"cflags": cflags_runtime,
"progress_category": "dolphin",
"host": False,
"objects": [
Object(Matching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/abort_exit.c"),
@@ -1185,6 +1231,7 @@ config.libs = [
"lib": "TRK_MINNOW_DOLPHIN",
"mw_version": "GC/1.3.2",
"cflags": cflags_runtime,
"progress_category": "dolphin",
"host": False,
"objects": [
Object(NonMatching, "TRK_MINNOW_DOLPHIN/Portable/mainloop.c"),
@@ -1217,6 +1264,7 @@ config.libs = [
"lib": "amcstubs",
"mw_version": "GC/1.3.2",
"cflags": cflags_runtime,
"progress_category": "dolphin",
"host": False,
"objects": [
Object(NonMatching, "amcstubs/AmcExi2Stubs.c"),
@@ -1226,6 +1274,7 @@ config.libs = [
"lib": "OdemuExi2",
"mw_version": "GC/1.3.2",
"cflags": cflags_runtime,
"progress_category": "dolphin",
"host": False,
"objects": [
Object(NonMatching, "OdemuExi2/DebuggerDriver.c"),
@@ -1235,6 +1284,7 @@ config.libs = [
"lib": "odenotstub",
"mw_version": "GC/1.3.2",
"cflags": cflags_runtime,
"progress_category": "dolphin",
"host": False,
"objects": [
Object(NonMatching, "odenotstub/odenotstub.c"),
@@ -1246,6 +1296,7 @@ config.libs = [
"lib": "REL",
"mw_version": "GC/1.3.2",
"cflags": cflags_rel,
"progress_category": "core",
"host": False,
"objects": [
Object(Matching, "REL/executor.c"),
tools/project.py
+364 -191
View File
@@ -29,24 +29,77 @@ if sys.platform == "cygwin":
f"\n(Current path: {sys.executable})"
)
Library = Dict[str, Any]
class Object:
def __init__(self, completed: bool, name: str, **options: Any) -> None:
self.name = name
self.base_name = Path(name).with_suffix("")
self.completed = completed
self.options: Dict[str, Any] = {
"add_to_all": True,
"add_to_all": None,
"asflags": None,
"extra_asflags": None,
"asm_dir": None,
"cflags": None,
"extra_asflags": None,
"extra_cflags": None,
"host": None,
"lib": None,
"mw_version": None,
"progress_category": None,
"shift_jis": None,
"source": name,
"src_dir": None,
}
self.options.update(options)
# Internal
self.src_path: Optional[Path] = None
self.asm_path: Optional[Path] = None
self.src_obj_path: Optional[Path] = None
self.asm_obj_path: Optional[Path] = None
self.host_obj_path: Optional[Path] = None
self.ctx_path: Optional[Path] = None
def resolve(self, config: "ProjectConfig", lib: Library) -> "Object":
# Use object options, then library options
obj = Object(self.completed, self.name, **lib)
for key, value in self.options.items():
if value is not None or key not in obj.options:
obj.options[key] = value
# Use default options from config
def set_default(key: str, value: Any) -> None:
if obj.options[key] is None:
obj.options[key] = value
set_default("add_to_all", True)
set_default("asflags", config.asflags)
set_default("asm_dir", config.asm_dir)
set_default("host", False)
set_default("mw_version", config.linker_version)
set_default("shift_jis", config.shift_jis)
set_default("src_dir", config.src_dir)
# Resolve paths
build_dir = config.out_path()
obj.src_path = Path(obj.options["src_dir"]) / obj.options["source"]
if obj.options["asm_dir"] is not None:
obj.asm_path = (
Path(obj.options["asm_dir"]) / obj.options["source"]
).with_suffix(".s")
base_name = Path(self.name).with_suffix("")
obj.src_obj_path = build_dir / "src" / f"{base_name}.o"
obj.asm_obj_path = build_dir / "mod" / f"{base_name}.o"
obj.host_obj_path = build_dir / "host" / f"{base_name}.o"
obj.ctx_path = build_dir / "src" / f"{base_name}.ctx"
return obj
class ProgressCategory:
def __init__(self, id: str, name: str) -> None:
self.id = id
self.name = name
class ProjectConfig:
def __init__(self) -> None:
@@ -69,6 +122,8 @@ class ProjectConfig:
self.wrapper: Optional[Path] = None # If None, download wibo on Linux
self.sjiswrap_tag: Optional[str] = None # Git tag
self.sjiswrap_path: Optional[Path] = None # If None, download
self.objdiff_tag: Optional[str] = None # Git tag
self.objdiff_path: Optional[Path] = None # If None, download
# Project config
self.non_matching: bool = False
@@ -79,21 +134,27 @@ class ProjectConfig:
self.generate_map: bool = False # Generate map file(s)
self.asflags: Optional[List[str]] = None # Assembler flags
self.ldflags: Optional[List[str]] = None # Linker flags
self.libs: Optional[List[Dict[str, Any]]] = None # List of libraries
self.libs: Optional[List[Library]] = None # List of libraries
self.linker_version: Optional[str] = None # mwld version
self.version: Optional[str] = None # Version name
self.warn_missing_config: bool = False # Warn on missing unit configuration
self.warn_missing_source: bool = False # Warn on missing source file
self.rel_strip_partial: bool = True # Generate PLFs with -strip_partial
self.rel_empty_file: Optional[
str
] = None # Object name for generating empty RELs
self.rel_empty_file: Optional[str] = (
None # Object name for generating empty RELs
)
self.shift_jis = (
True # Convert source files from UTF-8 to Shift JIS automatically
)
self.reconfig_deps: Optional[List[Path]] = (
None # Additional re-configuration dependency files
)
self.custom_build_rules: Optional[List[Dict[str, Any]]] = (
None # Custom ninja build rules
)
self.custom_build_steps: Optional[Dict[str, List[Dict[str, Any]]]] = (
None # Custom build steps, types are ["pre-compile", "post-compile", "post-link", "post-build"]
)
# Progress output and progress.json config
self.progress_all: bool = True # Include combined "all" category
@@ -101,6 +162,7 @@ class ProjectConfig:
self.progress_each_module: bool = (
True # Include individual modules, disable for large numbers of modules
)
self.progress_categories: List[ProgressCategory] = [] # Additional categories
# Progress fancy printing
self.progress_use_fancy: bool = False
@@ -125,12 +187,17 @@ class ProjectConfig:
if getattr(self, attr) is None:
sys.exit(f"ProjectConfig.{attr} missing")
def find_object(self, name: str) -> Optional[Tuple[Dict[str, Any], Object]]:
# Creates a map of object names to Object instances
# Options are fully resolved from the library and object
def objects(self) -> Dict[str, Object]:
out = {}
for lib in self.libs or {}:
for obj in lib["objects"]:
if obj.name == name:
return lib, obj
return None
objects: List[Object] = lib["objects"]
for obj in objects:
if obj.name in out:
sys.exit(f"Duplicate object name {obj.name}")
out[obj.name] = obj.resolve(self, lib)
return out
def out_path(self) -> Path:
return self.build_dir / str(self.version)
@@ -166,7 +233,7 @@ def load_build_config(
f = open(build_config_path, "r", encoding="utf-8")
build_config: Dict[str, Any] = json.load(f)
config_version = build_config.get("version")
if not config_version:
if config_version is None:
# Invalid config.json
f.close()
os.remove(build_config_path)
@@ -185,17 +252,19 @@ def load_build_config(
# Generate build.ninja and objdiff.json
def generate_build(config: ProjectConfig) -> None:
config.validate()
objects = config.objects()
build_config = load_build_config(config, config.out_path() / "config.json")
generate_build_ninja(config, build_config)
generate_objdiff_config(config, build_config)
generate_build_ninja(config, objects, build_config)
generate_objdiff_config(config, objects, build_config)
# Generate build.ninja
def generate_build_ninja(
config: ProjectConfig, build_config: Optional[Dict[str, Any]]
config: ProjectConfig,
objects: Dict[str, Object],
build_config: Optional[Dict[str, Any]],
) -> None:
config.validate()
out = io.StringIO()
n = ninja_syntax.Writer(out)
n.variable("ninja_required_version", "1.3")
@@ -219,7 +288,7 @@ def generate_build_ninja(
if config.debug:
ldflags += " -g"
n.variable("ldflags", ldflags)
if not config.linker_version:
if config.linker_version is None:
sys.exit("ProjectConfig.linker_version missing")
n.variable("mw_version", Path(config.linker_version))
n.newline()
@@ -231,6 +300,7 @@ def generate_build_ninja(
build_path = config.out_path()
progress_path = build_path / "progress.json"
report_path = build_path / "report.json"
build_tools_path = config.build_dir / "tools"
download_tool = config.tools_dir / "download_tool.py"
n.rule(
@@ -248,17 +318,27 @@ def generate_build_ninja(
deps="gcc",
)
cargo_rule_written = False
def write_cargo_rule():
nonlocal cargo_rule_written
if not cargo_rule_written:
n.pool("cargo", 1)
n.rule(
name="cargo",
command="cargo build --release --manifest-path $in --bin $bin --target-dir $target",
description="CARGO $bin",
pool="cargo",
depfile=Path("$target") / "release" / "$bin.d",
deps="gcc",
)
cargo_rule_written = True
if config.dtk_path is not None and config.dtk_path.is_file():
dtk = config.dtk_path
elif config.dtk_path is not None:
dtk = build_tools_path / "release" / f"dtk{EXE}"
n.rule(
name="cargo",
command="cargo build --release --manifest-path $in --bin $bin --target-dir $target",
description="CARGO $bin",
depfile=Path("$target") / "release" / "$bin.d",
deps="gcc",
)
write_cargo_rule()
n.build(
outputs=dtk,
rule="cargo",
@@ -283,6 +363,35 @@ def generate_build_ninja(
else:
sys.exit("ProjectConfig.dtk_tag missing")
if config.objdiff_path is not None and config.objdiff_path.is_file():
objdiff = config.objdiff_path
elif config.objdiff_path is not None:
objdiff = build_tools_path / "release" / f"objdiff-cli{EXE}"
write_cargo_rule()
n.build(
outputs=objdiff,
rule="cargo",
inputs=config.objdiff_path / "Cargo.toml",
implicit=config.objdiff_path / "Cargo.lock",
variables={
"bin": "objdiff-cli",
"target": build_tools_path,
},
)
elif config.objdiff_tag:
objdiff = build_tools_path / f"objdiff-cli{EXE}"
n.build(
outputs=objdiff,
rule="download_tool",
implicit=download_tool,
variables={
"tool": "objdiff-cli",
"tag": config.objdiff_tag,
},
)
else:
sys.exit("ProjectConfig.objdiff_tag missing")
if config.sjiswrap_path:
sjiswrap = config.sjiswrap_path
elif config.sjiswrap_tag:
@@ -361,6 +470,17 @@ def generate_build_ninja(
n.newline()
###
# Helper rule for downloading all tools
###
n.comment("Download all tools")
n.build(
outputs="tools",
rule="phony",
inputs=[dtk, sjiswrap, wrapper, compilers, binutils, objdiff],
)
n.newline()
###
# Build rules
###
@@ -443,6 +563,49 @@ def generate_build_ninja(
)
n.newline()
if len(config.custom_build_rules or {}) > 0:
n.comment("Custom project build rules (pre/post-processing)")
for rule in config.custom_build_rules or {}:
n.rule(
name=rule.get("name"),
command=rule.get("command"),
description=rule.get("description", None),
depfile=rule.get("depfile", None),
generator=rule.get("generator", False),
pool=rule.get("pool", None),
restat=rule.get("restat", False),
rspfile=rule.get("rspfile", None),
rspfile_content=rule.get("rspfile_content", None),
deps=rule.get("deps", None),
)
n.newline()
def write_custom_step(step: str) -> List[str]:
implicit = []
if config.custom_build_steps and step in config.custom_build_steps:
n.comment(f"Custom build steps ({step})")
for custom_step in config.custom_build_steps[step]:
outputs = custom_step.get("outputs")
if isinstance(outputs, list):
implicit.extend(outputs)
else:
implicit.append(outputs)
n.build(
outputs=outputs,
rule=custom_step.get("rule"),
inputs=custom_step.get("inputs", None),
implicit=custom_step.get("implicit", None),
order_only=custom_step.get("order_only", None),
variables=custom_step.get("variables", None),
implicit_outputs=custom_step.get("implicit_outputs", None),
pool=custom_step.get("pool", None),
dyndep=custom_step.get("dyndep", None),
)
n.newline()
return implicit
n.comment("Host build")
n.variable("host_cflags", "-I include -Wno-trigraphs")
n.variable(
@@ -461,14 +624,13 @@ def generate_build_ninja(
)
n.newline()
# Add all build steps needed before we compile (e.g. processing assets)
precompile_implicit = write_custom_step("pre-compile")
###
# Source files
###
n.comment("Source files")
build_asm_path = build_path / "mod"
build_src_path = build_path / "src"
build_host_path = build_path / "host"
build_config_path = build_path / "config.json"
def map_path(path: Path) -> Path:
return path.parent / (path.name + ".MAP")
@@ -511,16 +673,15 @@ def generate_build_ninja(
outputs=elf_path,
rule="link",
inputs=self.inputs,
implicit=[self.ldscript, *mwld_implicit],
implicit=[
*precompile_implicit,
self.ldscript,
*mwld_implicit,
*postcompile_implicit,
],
implicit_outputs=elf_map,
variables={"ldflags": elf_ldflags},
)
n.build(
outputs=dol_path,
rule="elf2dol",
inputs=elf_path,
implicit=dtk,
)
else:
preplf_path = build_path / self.name / f"{self.name}.preplf"
plf_path = build_path / self.name / f"{self.name}.plf"
@@ -565,95 +726,82 @@ def generate_build_ninja(
host_source_inputs: List[Path] = []
source_added: Set[Path] = set()
def c_build(
obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path
) -> Optional[Path]:
cflags_str = make_flags_str(options["cflags"])
if options["extra_cflags"] is not None:
extra_cflags_str = make_flags_str(options["extra_cflags"])
def c_build(obj: Object, src_path: Path) -> Optional[Path]:
cflags_str = make_flags_str(obj.options["cflags"])
if obj.options["extra_cflags"] is not None:
extra_cflags_str = make_flags_str(obj.options["extra_cflags"])
cflags_str += " " + extra_cflags_str
used_compiler_versions.add(options["mw_version"])
src_obj_path = build_src_path / f"{obj.base_name}.o"
src_base_path = build_src_path / obj.base_name
used_compiler_versions.add(obj.options["mw_version"])
# Avoid creating duplicate build rules
if src_obj_path in source_added:
return src_obj_path
source_added.add(src_obj_path)
shift_jis = options["shift_jis"]
if shift_jis is None:
shift_jis = config.shift_jis
if obj.src_obj_path in source_added:
return obj.src_obj_path
source_added.add(obj.src_obj_path)
# Add MWCC build rule
lib_name = obj.options["lib"]
n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})")
n.build(
outputs=src_obj_path,
rule="mwcc_sjis" if shift_jis else "mwcc",
outputs=obj.src_obj_path,
rule="mwcc_sjis" if obj.options["shift_jis"] else "mwcc",
inputs=src_path,
variables={
"mw_version": Path(options["mw_version"]),
"mw_version": Path(obj.options["mw_version"]),
"cflags": cflags_str,
"basedir": os.path.dirname(src_base_path),
"basefile": src_base_path,
"basedir": os.path.dirname(obj.src_obj_path),
"basefile": obj.src_obj_path.with_suffix(""),
},
implicit=mwcc_sjis_implicit if shift_jis else mwcc_implicit,
implicit=(
mwcc_sjis_implicit if obj.options["shift_jis"] else mwcc_implicit
),
)
# Add ctx build rule
ctx_path = build_src_path / f"{obj.base_name}.ctx"
n.build(
outputs=ctx_path,
outputs=obj.ctx_path,
rule="decompctx",
inputs=src_path,
implicit=decompctx,
)
# Add host build rule
if options.get("host", False):
host_obj_path = build_host_path / f"{obj.base_name}.o"
host_base_path = build_host_path / obj.base_name
if obj.options["host"]:
n.build(
outputs=host_obj_path,
outputs=obj.host_obj_path,
rule="host_cc" if src_path.suffix == ".c" else "host_cpp",
inputs=src_path,
variables={
"basedir": os.path.dirname(host_base_path),
"basefile": host_base_path,
"basedir": os.path.dirname(obj.host_obj_path),
"basefile": obj.host_obj_path.with_suffix(""),
},
)
if options["add_to_all"]:
host_source_inputs.append(host_obj_path)
if obj.options["add_to_all"]:
host_source_inputs.append(obj.host_obj_path)
n.newline()
if options["add_to_all"]:
source_inputs.append(src_obj_path)
if obj.options["add_to_all"]:
source_inputs.append(obj.src_obj_path)
return src_obj_path
return obj.src_obj_path
def asm_build(
obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path
) -> Optional[Path]:
asflags = options["asflags"] or config.asflags
if asflags is None:
def asm_build(obj: Object, src_path: Path, obj_path: Path) -> Optional[Path]:
if obj.options["asflags"] is None:
sys.exit("ProjectConfig.asflags missing")
asflags_str = make_flags_str(asflags)
if options["extra_asflags"] is not None:
extra_asflags_str = make_flags_str(options["extra_asflags"])
asflags_str = make_flags_str(obj.options["asflags"])
if obj.options["extra_asflags"] is not None:
extra_asflags_str = make_flags_str(obj.options["extra_asflags"])
asflags_str += " " + extra_asflags_str
asm_obj_path = build_asm_path / f"{obj.base_name}.o"
# Avoid creating duplicate build rules
if asm_obj_path in source_added:
return asm_obj_path
source_added.add(asm_obj_path)
if obj_path in source_added:
return obj_path
source_added.add(obj_path)
# Add assembler build rule
lib_name = obj.options["lib"]
n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})")
n.build(
outputs=asm_obj_path,
outputs=obj_path,
rule="as",
inputs=src_path,
variables={"asflags": asflags_str},
@@ -661,57 +809,40 @@ def generate_build_ninja(
)
n.newline()
if options["add_to_all"]:
source_inputs.append(asm_obj_path)
if obj.options["add_to_all"]:
source_inputs.append(obj_path)
return asm_obj_path
return obj_path
def add_unit(build_obj, link_step: LinkStep):
obj_path, obj_name = build_obj["object"], build_obj["name"]
result = config.find_object(obj_name)
if not result:
obj = objects.get(obj_name)
if obj is None:
if config.warn_missing_config and not build_obj["autogenerated"]:
print(f"Missing configuration for {obj_name}")
link_step.add(obj_path)
return
lib, obj = result
lib_name = lib["lib"]
# Use object options, then library options
options = lib.copy()
for key, value in obj.options.items():
if value is not None or key not in options:
options[key] = value
unit_src_path = Path(lib.get("src_dir", config.src_dir)) / options["source"]
unit_asm_path: Optional[Path] = None
if config.asm_dir is not None:
unit_asm_path = (
Path(lib.get("asm_dir", config.asm_dir)) / options["source"]
).with_suffix(".s")
link_built_obj = obj.completed
built_obj_path: Optional[Path] = None
if unit_src_path.exists():
if unit_src_path.suffix in (".c", ".cp", ".cpp"):
if obj.src_path.exists():
if obj.src_path.suffix in (".c", ".cp", ".cpp"):
# Add MWCC & host build rules
built_obj_path = c_build(obj, options, lib_name, unit_src_path)
elif unit_src_path.suffix == ".s":
built_obj_path = c_build(obj, obj.src_path)
elif obj.src_path.suffix == ".s":
# Add assembler build rule
built_obj_path = asm_build(obj, options, lib_name, unit_src_path)
built_obj_path = asm_build(obj, obj.src_path, obj.src_obj_path)
else:
sys.exit(f"Unknown source file type {unit_src_path}")
sys.exit(f"Unknown source file type {obj.src_path}")
else:
if config.warn_missing_source or obj.completed:
print(f"Missing source file {unit_src_path}")
print(f"Missing source file {obj.src_path}")
link_built_obj = False
# Assembly overrides
if unit_asm_path is not None and unit_asm_path.exists():
if obj.asm_path is not None and obj.asm_path.exists():
link_built_obj = True
built_obj_path = asm_build(obj, options, lib_name, unit_asm_path)
built_obj_path = asm_build(obj, obj.asm_path, obj.asm_obj_path)
if link_built_obj and built_obj_path is not None:
# Use the source-built object
@@ -720,7 +851,10 @@ def generate_build_ninja(
# Use the original (extracted) object
link_step.add(obj_path)
else:
sys.exit(f"Missing object for {obj_name}: {unit_src_path} {lib} {obj}")
lib_name = obj.options["lib"]
sys.exit(
f"Missing object for {obj_name}: {obj.src_path} {lib_name} {obj}"
)
# Add DOL link step
link_step = LinkStep(build_config)
@@ -736,7 +870,7 @@ def generate_build_ninja(
add_unit(unit, module_link_step)
# Add empty object to empty RELs
if len(module_link_step.inputs) == 0:
if not config.rel_empty_file:
if config.rel_empty_file is None:
sys.exit("ProjectConfig.rel_empty_file missing")
add_unit(
{
@@ -760,6 +894,9 @@ def generate_build_ninja(
if config.compilers_path and not os.path.exists(mw_path):
sys.exit(f"Linker {mw_path} does not exist")
# Add all build steps needed before we link and after compiling objects
postcompile_implicit = write_custom_step("post-compile")
###
# Link
###
@@ -768,6 +905,19 @@ def generate_build_ninja(
link_outputs.append(step.output())
n.newline()
# Add all build steps needed after linking and before GC/Wii native format generation
postlink_implicit = write_custom_step("post-link")
###
# Generate DOL
###
n.build(
outputs=link_steps[0].output(),
rule="elf2dol",
inputs=link_steps[0].partial_output(),
implicit=[*postlink_implicit, dtk],
)
###
# Generate RELs
###
@@ -830,6 +980,9 @@ def generate_build_ninja(
)
n.newline()
# Add all build steps needed post-build (re-building archives and such)
postbuild_implicit = write_custom_step("post-build")
###
# Helper rule for building all source files
###
@@ -867,7 +1020,7 @@ def generate_build_ninja(
outputs=ok_path,
rule="check",
inputs=config.check_sha_path,
implicit=[dtk, *link_outputs],
implicit=[dtk, *link_outputs, *postbuild_implicit],
)
n.newline()
@@ -886,6 +1039,21 @@ def generate_build_ninja(
implicit=[ok_path, configure_script, python_lib, config.config_path],
)
###
# Generate progress report
###
n.comment("Generate progress report")
n.rule(
name="report",
command=f"{objdiff} report generate -o $out",
description="REPORT",
)
n.build(
outputs=report_path,
rule="report",
implicit=[objdiff, "all_source"],
)
###
# Helper tools
###
@@ -932,6 +1100,7 @@ def generate_build_ninja(
###
# Split DOL
###
build_config_path = build_path / "config.json"
n.comment("Split DOL into relocatable objects")
n.rule(
name="split",
@@ -967,7 +1136,7 @@ def generate_build_ninja(
configure_script,
python_lib,
python_lib_dir / "ninja_syntax.py",
*(config.reconfig_deps or [])
*(config.reconfig_deps or []),
],
)
n.newline()
@@ -992,9 +1161,11 @@ def generate_build_ninja(
# Generate objdiff.json
def generate_objdiff_config(
config: ProjectConfig, build_config: Optional[Dict[str, Any]]
config: ProjectConfig,
objects: Dict[str, Object],
build_config: Optional[Dict[str, Any]],
) -> None:
if not build_config:
if build_config is None:
return
objdiff_config: Dict[str, Any] = {
@@ -1063,30 +1234,12 @@ def generate_objdiff_config(
"target_path": obj_path,
}
result = config.find_object(obj_name)
if not result:
obj = objects.get(obj_name)
if obj is None or not obj.src_path.exists():
objdiff_config["units"].append(unit_config)
return
lib, obj = result
src_dir = Path(lib.get("src_dir", config.src_dir))
# Use object options, then library options
options = lib.copy()
for key, value in obj.options.items():
if value is not None or key not in options:
options[key] = value
unit_src_path = src_dir / str(options["source"])
if not unit_src_path.exists():
objdiff_config["units"].append(unit_config)
return
cflags = options["cflags"]
src_obj_path = build_path / "src" / f"{obj.base_name}.o"
src_ctx_path = build_path / "src" / f"{obj.base_name}.ctx"
cflags = obj.options["cflags"]
reverse_fn_order = False
if type(cflags) is list:
for flag in cflags:
@@ -1105,29 +1258,38 @@ def generate_objdiff_config(
cflags = list(filter(keep_flag, cflags))
# Add appropriate lang flag
if unit_src_path.suffix in (".cp", ".cpp"):
if obj.src_path.suffix in (".cp", ".cpp"):
cflags.insert(0, "-lang=c++")
else:
cflags.insert(0, "-lang=c")
unit_config["base_path"] = src_obj_path
unit_config["base_path"] = obj.src_obj_path
unit_config["reverse_fn_order"] = reverse_fn_order
unit_config["complete"] = obj.completed
compiler_version = COMPILER_MAP.get(options["mw_version"])
compiler_version = COMPILER_MAP.get(obj.options["mw_version"])
if compiler_version is None:
print(f"Missing scratch compiler mapping for {options['mw_version']}")
print(f"Missing scratch compiler mapping for {obj.options['mw_version']}")
else:
cflags_str = make_flags_str(cflags)
if options["extra_cflags"] is not None:
extra_cflags_str = make_flags_str(options["extra_cflags"])
if obj.options["extra_cflags"] is not None:
extra_cflags_str = make_flags_str(obj.options["extra_cflags"])
cflags_str += " " + extra_cflags_str
unit_config["scratch"] = {
"platform": "gc_wii",
"compiler": compiler_version,
"c_flags": cflags_str,
"ctx_path": src_ctx_path,
"ctx_path": obj.ctx_path,
"build_ctx": True,
}
metadata = {
"source_path": obj.src_path,
}
if obj.options["progress_category"] is not None:
if isinstance(obj.options["progress_category"], list):
metadata["progress_categories"] = obj.options["progress_category"]
else:
metadata["progress_categories"] = [obj.options["progress_category"]]
unit_config["metadata"] = metadata
objdiff_config["units"].append(unit_config)
# Add DOL units
@@ -1150,9 +1312,11 @@ def generate_objdiff_config(
# Calculate, print and write progress to progress.json
def calculate_progress(config: ProjectConfig) -> None:
config.validate()
objects = config.objects()
out_path = config.out_path()
build_config = load_build_config(config, out_path / "config.json")
if not build_config:
if build_config is None:
return
class ProgressUnit:
@@ -1184,12 +1348,8 @@ def calculate_progress(config: ProjectConfig) -> None:
# Skip autogenerated objects
return
result = config.find_object(build_obj["name"])
if not result:
return
_, obj = result
if not obj.completed:
obj = objects.get(build_obj["name"])
if obj is None or not obj.completed:
return
self.code_progress += build_obj["code_size"]
@@ -1203,26 +1363,53 @@ def calculate_progress(config: ProjectConfig) -> None:
def data_frac(self) -> float:
return self.data_progress / self.data_total
progress_units: Dict[str, ProgressUnit] = {}
if len(config.progress_categories) > 0:
for category in config.progress_categories:
progress_units[category.id] = ProgressUnit(category.name)
else:
if config.progress_all:
progress_units["all"] = ProgressUnit("All")
progress_units["dol"] = ProgressUnit("DOL")
if len(build_config["modules"]) > 0:
if config.progress_modules:
progress_units["modules"] = ProgressUnit("Modules")
if config.progress_each_module:
for module in build_config["modules"]:
progress_units[module["name"]] = ProgressUnit(module["name"])
def add_unit(id: str, unit: Dict[str, Any]) -> None:
progress = progress_units.get(id)
if progress is not None:
progress.add(unit)
# Add DOL units
all_progress = ProgressUnit("All") if config.progress_all else None
dol_progress = ProgressUnit("DOL")
for unit in build_config["units"]:
if all_progress:
all_progress.add(unit)
dol_progress.add(unit)
add_unit("all", unit)
add_unit("dol", unit)
obj = objects.get(unit["name"])
if obj is not None:
category = obj.options["progress_category"]
if isinstance(category, list):
for category in category:
add_unit(category, unit)
elif category is not None:
add_unit(category, unit)
# Add REL units
rels_progress = ProgressUnit("Modules") if config.progress_modules else None
modules_progress: List[ProgressUnit] = []
for module in build_config["modules"]:
progress = ProgressUnit(module["name"])
modules_progress.append(progress)
for unit in module["units"]:
if all_progress:
all_progress.add(unit)
if rels_progress:
rels_progress.add(unit)
progress.add(unit)
add_unit("all", unit)
add_unit("modules", unit)
add_unit(module["name"], unit)
obj = objects.get(unit["name"])
if obj is not None:
category = obj.options["progress_category"]
if isinstance(category, list):
for category in category:
add_unit(category, unit)
elif category is not None:
add_unit(category, unit)
# Print human-readable progress
print("Progress:")
@@ -1250,15 +1437,8 @@ def calculate_progress(config: ProjectConfig) -> None:
)
)
if all_progress:
print_category(all_progress)
print_category(dol_progress)
module_count = len(build_config["modules"])
if module_count > 0:
print_category(rels_progress)
if config.progress_each_module:
for progress in modules_progress:
print_category(progress)
for progress in progress_units.values():
print_category(progress)
# Generate and write progress.json
progress_json: Dict[str, Any] = {}
@@ -1271,14 +1451,7 @@ def calculate_progress(config: ProjectConfig) -> None:
"data/total": unit.data_total,
}
if all_progress:
add_category("all", all_progress)
add_category("dol", dol_progress)
if len(build_config["modules"]) > 0:
if rels_progress:
add_category("modules", rels_progress)
if config.progress_each_module:
for progress in modules_progress:
add_category(progress.name, progress)
for id, progress in progress_units.items():
add_category(id, progress)
with open(out_path / "progress.json", "w", encoding="utf-8") as w:
json.dump(progress_json, w, indent=4)