mirror/tww
1
0
Fork 0
mirror of https://github.com/zeldaret/tww.git synced 2026-05-22 22:44:17 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update dtk-template, dtk, and objdiff

Browse Source
This commit is contained in:
LagoLunatic
2025-01-28 14:01:04 -05:00
parent 901bc51f40
commit d6c3081559
7 changed files with 206 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/GZLE01/splits.txt
+6 -6
View File
@@ -141,9 +141,6 @@ m_Do/m_Do_machine_exception.cpp:
.rodata start:0x803396F8 end:0x803398D8
.sbss start:0x803F69B0 end:0x803F69B8
dolphin/os/__start.c:
.init start:0x80003100 end:0x800033A8
c/c_damagereaction.cpp:
.text start:0x8001BEDC end:0x800227A0
.ctors start:0x803386A4 end:0x803386A8
@@ -1704,9 +1701,6 @@ d/d_wpot_water.cpp:
.sbss start:0x803F7300 end:0x803F7308
.sdata2 start:0x803FBB00 end:0x803FBB20
PowerPC_EABI_Support/Runtime/Src/__mem.c:
.init start:0x800033A8 end:0x800034E0
DynamicLink.cpp:
.text start:0x80240098 end:0x80241278
.text start:0x80241278 end:0x802412C4
@@ -4008,6 +4002,12 @@ TRK_MINNOW_DOLPHIN/Portable/notify.c:
TRK_MINNOW_DOLPHIN/ppc/Generic/flush_cache.c:
.text start:0x80335690 end:0x803356C8
dolphin/os/__start.c:
.init start:0x80003100 end:0x800033A8
PowerPC_EABI_Support/Runtime/Src/__mem.c:
.init start:0x800033A8 end:0x800034E0
TRK_MINNOW_DOLPHIN/Portable/mem_TRK.c:
.init start:0x800034E0 end:0x80003534
.text start:0x803356C8 end:0x80335784
config/GZLJ01/splits.txt
+6 -6
View File
@@ -140,9 +140,6 @@ m_Do/m_Do_machine_exception.cpp:
.rodata start:0x80337158 end:0x80337338
.sbss start:0x803E9E90 end:0x803E9E98
dolphin/os/__start.c:
.init start:0x80003100 end:0x800033A8
c/c_damagereaction.cpp:
.text start:0x8001BB70 end:0x80022430
.ctors start:0x803360A0 end:0x803360A4
@@ -1704,9 +1701,6 @@ d/d_wpot_water.cpp:
.sbss start:0x803EA7A0 end:0x803EA7A8
.sdata2 start:0x803EEFD8 end:0x803EEFF8
PowerPC_EABI_Support/Runtime/Src/__mem.c:
.init start:0x800033A8 end:0x800034E0
DynamicLink.cpp:
.text start:0x8023D828 end:0x8023EA08
.text start:0x8023EA08 end:0x8023EA54
@@ -4004,6 +3998,12 @@ TRK_MINNOW_DOLPHIN/Portable/notify.c:
TRK_MINNOW_DOLPHIN/ppc/Generic/flush_cache.c:
.text start:0x80333094 end:0x803330CC
dolphin/os/__start.c:
.init start:0x80003100 end:0x800033A8
PowerPC_EABI_Support/Runtime/Src/__mem.c:
.init start:0x800033A8 end:0x800034E0
TRK_MINNOW_DOLPHIN/Portable/mem_TRK.c:
.init start:0x800034E0 end:0x80003534
.text start:0x803330CC end:0x80333188
config/GZLP01/splits.txt
+6 -6
View File
@@ -141,9 +141,6 @@ m_Do/m_Do_machine_exception.cpp:
.rodata start:0x8033EC90 end:0x8033EE70
.sbss start:0x803FE1B0 end:0x803FE1B8
dolphin/os/__start.c:
.init start:0x80003100 end:0x800033A8
c/c_damagereaction.cpp:
.text start:0x8001C22C end:0x80022AF0
.ctors start:0x8033DBE4 end:0x8033DBE8
@@ -1704,9 +1701,6 @@ d/d_wpot_water.cpp:
.sbss start:0x803FEB20 end:0x803FEB28
.sdata2 start:0x80403350 end:0x80403370
PowerPC_EABI_Support/Runtime/Src/__mem.c:
.init start:0x800033A8 end:0x800034E0
DynamicLink.cpp:
.text start:0x80245238 end:0x80246418
.text start:0x80246418 end:0x80246464
@@ -4008,6 +4002,12 @@ TRK_MINNOW_DOLPHIN/Portable/notify.c:
TRK_MINNOW_DOLPHIN/ppc/Generic/flush_cache.c:
.text start:0x8033ABE8 end:0x8033AC20
dolphin/os/__start.c:
.init start:0x80003100 end:0x800033A8
PowerPC_EABI_Support/Runtime/Src/__mem.c:
.init start:0x800033A8 end:0x800034E0
TRK_MINNOW_DOLPHIN/Portable/mem_TRK.c:
.init start:0x800034E0 end:0x80003534
.text start:0x8033AC20 end:0x8033ACDC
configure.py
+19 -2
View File
@@ -151,8 +151,8 @@ if args.no_asm:
# Tool versions
config.binutils_tag = "2.42-1"
config.compilers_tag = "20240706"
config.dtk_tag = "v1.3.0"
config.objdiff_tag = "v2.4.0"
config.dtk_tag = "v1.4.0"
config.objdiff_tag = "v2.7.1"
config.sjiswrap_tag = "v1.2.0"
config.wibo_tag = "0.6.11"
@@ -1799,6 +1799,23 @@ if config_path.exists():
emit_build_rule(asset)
# Optional callback to adjust link order. This can be used to add, remove, or reorder objects.
# This is called once per module, with the module ID and the current link order.
#
# For example, this adds "dummy.c" to the end of the DOL link order if configured with --non-matching.
# "dummy.c" *must* be configured as a Matching (or Equivalent) object in order to be linked.
def link_order_callback(module_id: int, objects: List[str]) -> List[str]:
# Don't modify the link order for matching builds
if not config.non_matching:
return objects
if module_id == 0: # DOL
return objects + ["dummy.c"]
return objects
# Uncomment to enable the link order callback.
# config.link_order_callback = link_order_callback
# Optional extra categories for progress tracking
config.progress_categories = [
ProgressCategory("game", "TWW Game Code"),
tools/decompctx.py
+31 -22
View File
@@ -18,63 +18,63 @@ from typing import List
script_dir = os.path.dirname(os.path.realpath(__file__))
root_dir = os.path.abspath(os.path.join(script_dir, ".."))
src_dir = os.path.join(root_dir, "src")
include_dirs = [
os.path.join(root_dir, "include"),
os.path.join(root_dir, "src"),
os.path.join(root_dir, "src/PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Include"),
os.path.join(root_dir, "src/PowerPC_EABI_Support/MSL/MSL_C/MSL_Common_Embedded/Math/Include"),
os.path.join(root_dir, "src/PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/Include"),
os.path.join(root_dir, "src/PowerPC_EABI_Support/MSL/MSL_C++/MSL_Common/Include"),
os.path.join(root_dir, "src/PowerPC_EABI_Support/Runtime/Inc"),
]
include_dirs: List[str] = [] # Set with -I flag
include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]')
guard_pattern = re.compile(r"^#ifndef\s+(.*)$")
include_pattern = re.compile(r'^#\s*include\s*[<"](.+?)[>"]')
guard_pattern = re.compile(r"^#\s*ifndef\s+(.*)$")
once_pattern = re.compile(r"^#\s*pragma\s+once$")
defines = set()
deps = []
def import_h_file(in_file: str, r_path: str, deps: List[str]) -> str:
def import_h_file(in_file: str, r_path: str) -> str:
rel_path = os.path.join(root_dir, r_path, in_file)
if os.path.exists(rel_path):
return import_c_file(rel_path, deps)
return import_c_file(rel_path)
for include_dir in include_dirs:
inc_path = os.path.join(include_dir, in_file)
if os.path.exists(inc_path):
return import_c_file(inc_path, deps)
return import_c_file(inc_path)
else:
print("Failed to locate", in_file)
return ""
def import_c_file(in_file: str, deps: List[str]) -> str:
def import_c_file(in_file: str) -> str:
in_file = os.path.relpath(in_file, root_dir)
deps.append(in_file)
out_text = ""
try:
with open(in_file, encoding="utf-8") as file:
out_text += process_file(in_file, list(file), deps)
out_text += process_file(in_file, list(file))
except Exception:
with open(in_file) as file:
out_text += process_file(in_file, list(file), deps)
out_text += process_file(in_file, list(file))
return out_text
def process_file(in_file: str, lines: List[str], deps: List[str]) -> str:
def process_file(in_file: str, lines: List[str]) -> str:
out_text = ""
for idx, line in enumerate(lines):
guard_match = guard_pattern.match(line.strip())
if idx == 0:
guard_match = guard_pattern.match(line.strip())
if guard_match:
if guard_match[1] in defines:
break
defines.add(guard_match[1])
else:
once_match = once_pattern.match(line.strip())
if once_match:
if in_file in defines:
break
defines.add(in_file)
print("Processing file", in_file)
include_match = include_pattern.match(line.strip())
if include_match and not include_match[1].endswith(".s"):
out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n'
out_text += import_h_file(include_match[1], os.path.dirname(in_file), deps)
out_text += import_h_file(include_match[1], os.path.dirname(in_file))
out_text += f'/* end "{include_match[1]}" */\n'
else:
out_text += line
@@ -105,10 +105,19 @@ def main():
"--depfile",
help="""Dependency file""",
)
parser.add_argument(
"-I",
"--include",
help="""Include directory""",
action="append",
)
args = parser.parse_args()
deps = []
output = import_c_file(args.c_file, deps)
if args.include is None:
exit("No include directories specified")
global include_dirs
include_dirs = args.include
output = import_c_file(args.c_file)
with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f:
f.write(output)
tools/download_tool.py
+30 -16
View File
@@ -91,6 +91,21 @@ TOOLS: Dict[str, Callable[[str], str]] = {
"wibo": wibo_url,
}
def download(url, response, output) -> None:
if url.endswith(".zip"):
data = io.BytesIO(response.read())
with zipfile.ZipFile(data) as f:
f.extractall(output)
# Make all files executable
for root, _, files in os.walk(output):
for name in files:
os.chmod(os.path.join(root, name), 0o755)
output.touch(mode=0o755) # Update dir modtime
else:
with open(output, "wb") as f:
shutil.copyfileobj(response, f)
st = os.stat(output)
os.chmod(output, st.st_mode | stat.S_IEXEC)
def main() -> None:
parser = argparse.ArgumentParser()
@@ -104,22 +119,21 @@ def main() -> None:
print(f"Downloading {url} to {output}")
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
with urllib.request.urlopen(req) as response:
if url.endswith(".zip"):
data = io.BytesIO(response.read())
with zipfile.ZipFile(data) as f:
f.extractall(output)
# Make all files executable
for root, _, files in os.walk(output):
for name in files:
os.chmod(os.path.join(root, name), 0o755)
output.touch(mode=0o755) # Update dir modtime
else:
with open(output, "wb") as f:
shutil.copyfileobj(response, f)
st = os.stat(output)
os.chmod(output, st.st_mode | stat.S_IEXEC)
try:
with urllib.request.urlopen(req) as response:
download(url, response, output)
except urllib.error.URLError as e:
if str(e).find("CERTIFICATE_VERIFY_FAILED") == -1:
raise e
try:
import certifi
import ssl
except:
print("\"certifi\" module not found. Please install it using \"python -m pip install certifi\".")
return
with urllib.request.urlopen(req, context=ssl.create_default_context(cafile=certifi.where())) as response:
download(url, response, output)
if __name__ == "__main__":
main()
tools/project.py
+108 -37
View File
@@ -17,7 +17,20 @@ import os
import platform
import sys
from pathlib import Path
from typing import IO, Any, Dict, Iterable, List, Optional, Set, Tuple, Union, cast
from typing import (
Any,
Callable,
cast,
Dict,
IO,
Iterable,
List,
Optional,
Set,
Tuple,
TypedDict,
Union,
)
from . import ninja_syntax
from .ninja_syntax import serialize_path
@@ -176,7 +189,12 @@ class ProjectConfig:
True # Generate compile_commands.json for clangd
)
self.extra_clang_flags: List[str] = [] # Extra flags for clangd
self.scratch_preset_id: Optional[int] = None # Default decomp.me preset ID for scratches
self.scratch_preset_id: Optional[int] = (
None # Default decomp.me preset ID for scratches
)
self.link_order_callback: Optional[Callable[[int, List[str]], List[str]]] = (
None # Callback to add/remove/reorder units within a module
)
# Progress output, progress.json and report.json config
self.progress = True # Enable report.json generation and CLI progress output
@@ -292,10 +310,38 @@ def make_flags_str(flags: Optional[List[str]]) -> str:
return " ".join(flags)
# Unit configuration
class BuildConfigUnit(TypedDict):
object: Optional[str]
name: str
autogenerated: bool
# Module configuration
class BuildConfigModule(TypedDict):
name: str
module_id: int
ldscript: str
entry: str
units: List[BuildConfigUnit]
# Module link configuration
class BuildConfigLink(TypedDict):
modules: List[str]
# Build configuration generated by decomp-toolkit
class BuildConfig(BuildConfigModule):
version: str
modules: List[BuildConfigModule]
links: List[BuildConfigLink]
# Load decomp-toolkit generated config.json
def load_build_config(
config: ProjectConfig, build_config_path: Path
) -> Optional[Dict[str, Any]]:
) -> Optional[BuildConfig]:
if not build_config_path.is_file():
return None
@@ -303,7 +349,7 @@ def load_build_config(
return tuple(map(int, (v.split("."))))
f = open(build_config_path, "r", encoding="utf-8")
build_config: Dict[str, Any] = json.load(f)
build_config: BuildConfig = json.load(f)
config_version = build_config.get("version")
if config_version is None:
print("Invalid config.json, regenerating...")
@@ -319,6 +365,24 @@ def load_build_config(
return None
f.close()
# Apply link order callback
if config.link_order_callback:
modules: List[BuildConfigModule] = [build_config, *build_config["modules"]]
for module in modules:
unit_names = list(map(lambda u: u["name"], module["units"]))
unit_names = config.link_order_callback(module["module_id"], unit_names)
units: List[BuildConfigUnit] = []
for unit_name in unit_names:
units.append(
# Find existing unit or create a new one
next(
(u for u in module["units"] if u["name"] == unit_name),
{"object": None, "name": unit_name, "autogenerated": False},
)
)
module["units"] = units
return build_config
@@ -336,7 +400,7 @@ def generate_build(config: ProjectConfig) -> None:
def generate_build_ninja(
config: ProjectConfig,
objects: Dict[str, Object],
build_config: Optional[Dict[str, Any]],
build_config: Optional[BuildConfig],
) -> None:
out = io.StringIO()
n = ninja_syntax.Writer(out)
@@ -380,7 +444,7 @@ def generate_build_ninja(
decompctx = config.tools_dir / "decompctx.py"
n.rule(
name="decompctx",
command=f"$python {decompctx} $in -o $out -d $out.d",
command=f"$python {decompctx} $in -o $out -d $out.d $includes",
description="CTX $in",
depfile="$out.d",
deps="gcc",
@@ -697,9 +761,9 @@ def generate_build_ninja(
return path.parent / (path.name + ".MAP")
class LinkStep:
def __init__(self, config: Dict[str, Any]) -> None:
self.name: str = config["name"]
self.module_id: int = config["module_id"]
def __init__(self, config: BuildConfigModule) -> None:
self.name = config["name"]
self.module_id = config["module_id"]
self.ldscript: Optional[Path] = Path(config["ldscript"])
self.entry = config["entry"]
self.inputs: List[str] = []
@@ -809,10 +873,8 @@ def generate_build_ninja(
else:
extra_cflags.insert(0, "-lang=c")
cflags_str = make_flags_str(cflags)
if len(extra_cflags) > 0:
extra_cflags_str = make_flags_str(extra_cflags)
cflags_str += " " + extra_cflags_str
all_cflags = cflags + extra_cflags
cflags_str = make_flags_str(all_cflags)
used_compiler_versions.add(obj.options["mw_version"])
# Add MWCC build rule
@@ -836,11 +898,21 @@ def generate_build_ninja(
# Add ctx build rule
if obj.ctx_path is not None:
include_dirs = []
for flag in all_cflags:
if (
flag.startswith("-i ")
or flag.startswith("-I ")
or flag.startswith("-I+")
):
include_dirs.append(flag[3:])
includes = " ".join([f"-I {d}" for d in include_dirs])
n.build(
outputs=obj.ctx_path,
rule="decompctx",
inputs=src_path,
implicit=decompctx,
variables={"includes": includes},
)
# Add host build rule
@@ -897,13 +969,14 @@ def generate_build_ninja(
return obj_path
def add_unit(build_obj, link_step: LinkStep):
def add_unit(build_obj: BuildConfigUnit, link_step: LinkStep):
obj_path, obj_name = build_obj["object"], build_obj["name"]
obj = objects.get(obj_name)
if obj is None:
if config.warn_missing_config and not build_obj["autogenerated"]:
print(f"Missing configuration for {obj_name}")
link_step.add(obj_path)
if obj_path is not None:
link_step.add(Path(obj_path))
return
link_built_obj = obj.completed
@@ -932,12 +1005,7 @@ def generate_build_ninja(
link_step.add(built_obj_path)
elif obj_path is not None:
# Use the original (extracted) object
link_step.add(obj_path)
else:
lib_name = obj.options["lib"]
sys.exit(
f"Missing object for {obj_name}: {obj.src_path} {lib_name} {obj}"
)
link_step.add(Path(obj_path))
# Add DOL link step
link_step = LinkStep(build_config)
@@ -1258,7 +1326,7 @@ def generate_build_ninja(
def generate_objdiff_config(
config: ProjectConfig,
objects: Dict[str, Object],
build_config: Optional[Dict[str, Any]],
build_config: Optional[BuildConfig],
) -> None:
if build_config is None:
return
@@ -1323,7 +1391,7 @@ def generate_objdiff_config(
}
def add_unit(
build_obj: Dict[str, Any], module_name: str, progress_categories: List[str]
build_obj: BuildConfigUnit, module_name: str, progress_categories: List[str]
) -> None:
obj_path, obj_name = build_obj["object"], build_obj["name"]
base_object = Path(obj_name).with_suffix("")
@@ -1358,9 +1426,21 @@ def generate_objdiff_config(
unit_config["base_path"] = obj.src_obj_path
unit_config["metadata"]["source_path"] = obj.src_path
cflags = obj.options["cflags"]
# Filter out include directories
def keep_flag(flag):
return (
not flag.startswith("-i ")
and not flag.startswith("-i-")
and not flag.startswith("-I ")
and not flag.startswith("-I+")
and not flag.startswith("-I-")
)
all_cflags = list(
filter(keep_flag, obj.options["cflags"] + obj.options["extra_cflags"])
)
reverse_fn_order = False
for flag in cflags:
for flag in all_cflags:
if not flag.startswith("-inline "):
continue
for value in flag.split(" ")[1].split(","):
@@ -1369,20 +1449,11 @@ def generate_objdiff_config(
elif value == "nodeferred":
reverse_fn_order = False
# Filter out include directories
def keep_flag(flag):
return not flag.startswith("-i ") and not flag.startswith("-I ")
cflags = list(filter(keep_flag, cflags))
compiler_version = COMPILER_MAP.get(obj.options["mw_version"])
if compiler_version is None:
print(f"Missing scratch compiler mapping for {obj.options['mw_version']}")
else:
cflags_str = make_flags_str(cflags)
if len(obj.options["extra_cflags"]) > 0:
extra_cflags_str = make_flags_str(obj.options["extra_cflags"])
cflags_str += " " + extra_cflags_str
cflags_str = make_flags_str(all_cflags)
unit_config["scratch"] = {
"platform": "gc_wii",
"compiler": compiler_version,
@@ -1468,7 +1539,7 @@ def generate_objdiff_config(
def generate_compile_commands(
config: ProjectConfig,
objects: Dict[str, Object],
build_config: Optional[Dict[str, Any]],
build_config: Optional[BuildConfig],
) -> None:
if build_config is None or not config.generate_compile_commands:
return
@@ -1557,7 +1628,7 @@ def generate_compile_commands(
clangd_config = []
def add_unit(build_obj: Dict[str, Any]) -> None:
def add_unit(build_obj: BuildConfigUnit) -> None:
obj = objects.get(build_obj["name"])
if obj is None:
return