mirror/ss
1
0
Fork 0
mirror of https://github.com/zeldaret/ss synced 2026-09-08 03:17:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

dtk upgrade

Browse Source
This commit is contained in:
robojumper
2025-05-19 00:33:25 +02:00
parent 924db04bd9
commit f0ccf5093e
544 changed files with 10165 additions and 9847 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/changes_fmt.py
+154
View File
@@ -0,0 +1,154 @@
#!/usr/bin/env python3
from argparse import ArgumentParser
import os
import json
from pathlib import Path
from typing import Tuple
script_dir = os.path.dirname(os.path.realpath(__file__))
root_dir = os.path.abspath(os.path.join(script_dir, ".."))
UNIT_KEYS_TO_DIFF = [
"fuzzy_match_percent",
"matched_code_percent",
"matched_data_percent",
"complete_code_percent",
"complete_data_percent",
]
FUNCTION_KEYS_TO_DIFF = [
"fuzzy_match_percent",
]
Change = Tuple[str, str, float, float]
def get_changes(changes_file: str) -> list[Change]:
changes_file = os.path.relpath(changes_file, root_dir)
with open(changes_file, "r") as f:
changes_json = json.load(f)
regressions = []
progressions = []
def diff_key(object_name: str, object: dict, key: str):
from_value = object.get("from", {}).get(key, 0.0)
to_value = object.get("to", {}).get(key, 0.0)
key = key.removesuffix("_percent")
change = (object_name, key, from_value, to_value)
if from_value > to_value:
regressions.append(change)
elif to_value > from_value:
progressions.append(change)
for key in UNIT_KEYS_TO_DIFF:
diff_key(None, changes_json, key)
for unit in changes_json.get("units", []):
unit_name = unit["name"]
for key in UNIT_KEYS_TO_DIFF:
diff_key(unit_name, unit, key)
# Ignore sections
for func in unit.get("functions", []):
func_name = func["name"]
for key in FUNCTION_KEYS_TO_DIFF:
diff_key(func_name, func, key)
return regressions, progressions
def generate_changes_plaintext(changes: list[Change]) -> str:
if len(changes) == 0:
return ""
table_total_width = 136
percents_max_len = 7 + 4 + 7
key_max_len = max(len(key) for _, key, _, _ in changes)
name_max_len = max(len(name or "Total") for name, _, _, _ in changes)
max_width_for_name_col = table_total_width - 3 - key_max_len - 3 - percents_max_len
name_max_len = min(max_width_for_name_col, name_max_len)
out_lines = []
for name, key, from_value, to_value in changes:
if name is None:
name = "Total"
if len(name) > name_max_len:
name = name[: name_max_len - len("[...]")] + "[...]"
out_lines.append(
f"{name:>{name_max_len}} | {key:<{key_max_len}} | {from_value:6.2f}% -> {to_value:5.2f}%"
)
return "\n".join(out_lines)
def generate_changes_markdown(changes: list[Change], description: str) -> str:
if len(changes) == 0:
return ""
out_lines = []
name_max_len = 100
out_lines.append("<details>")
out_lines.append(
f"<summary>Detected {len(changes)} {description} compared to the base:</summary>"
)
out_lines.append("") # Must include a blank line before a table
out_lines.append("| Name | Type | Before | After |")
out_lines.append("| ---- | ---- | ------ | ----- |")
for name, key, from_value, to_value in changes:
if name is None:
name = "Total"
else:
if len(name) > name_max_len:
name = name[: name_max_len - len("...")] + "..."
name = f"`{name}`" # Surround with backticks
key = key.replace("_", " ").capitalize()
out_lines.append(f"| {name} | {key} | {from_value:.2f}% | {to_value:.2f}% |")
out_lines.append("</details>")
return "\n".join(out_lines)
def main():
parser = ArgumentParser(description="Format objdiff-cli report changes.")
parser.add_argument(
"report_changes_file",
type=Path,
help="""path to the JSON file containing the changes, generated by objdiff-cli.""",
)
parser.add_argument(
"-o",
"--output",
type=Path,
help="""Output file (prints to console if unspecified)""",
)
parser.add_argument(
"--all",
action="store_true",
help="""Includes progressions as well.""",
)
args = parser.parse_args()
regressions, progressions = get_changes(args.report_changes_file)
if args.output:
markdown_output = generate_changes_markdown(regressions, "regressions")
if args.all:
markdown_output += generate_changes_markdown(progressions, "progressions")
with open(args.output, "w", encoding="utf-8") as f:
f.write(markdown_output)
else:
if args.all:
changes = progressions + regressions
else:
changes = regressions
text_output = generate_changes_plaintext(changes)
print(text_output)
if __name__ == "__main__":
main()
tools/project.py
+85 -39
View File
@@ -196,9 +196,8 @@ class ProjectConfig:
None # Callback to add/remove/reorder units within a module
)
# Progress output, progress.json and report.json config
# Progress output and report.json config
self.progress = True # Enable report.json generation and CLI progress output
self.progress_all: bool = True # Include combined "all" category
self.progress_modules: bool = True # Include combined "modules" category
self.progress_each_module: bool = (
False # Include individual modules, disable for large numbers of modules
@@ -207,6 +206,9 @@ class ProjectConfig:
self.print_progress_categories: Union[bool, List[str]] = (
True # Print additional progress categories in the CLI progress output
)
self.progress_report_args: Optional[List[str]] = (
None # Flags to `objdiff-cli report generate`
)
# Progress fancy printing
self.progress_use_fancy: bool = False
@@ -423,6 +425,7 @@ def generate_build_ninja(
if config.linker_version is None:
sys.exit("ProjectConfig.linker_version missing")
n.variable("mw_version", Path(config.linker_version))
n.variable("objdiff_report_args", make_flags_str(config.progress_report_args))
n.newline()
###
@@ -431,7 +434,6 @@ def generate_build_ninja(
n.comment("Tooling")
build_path = config.out_path()
progress_path = build_path / "progress.json"
report_path = build_path / "report.json"
build_tools_path = config.build_dir / "tools"
download_tool = config.tools_dir / "download_tool.py"
@@ -1188,7 +1190,7 @@ def generate_build_ninja(
description="PROGRESS",
)
n.build(
outputs=progress_path,
outputs="progress",
rule="progress",
implicit=[
ok_path,
@@ -1205,7 +1207,7 @@ def generate_build_ninja(
n.comment("Generate progress report")
n.rule(
name="report",
command=f"{objdiff} report generate -o $out",
command=f"{objdiff} report generate $objdiff_report_args -o $out",
description="REPORT",
)
n.build(
@@ -1215,6 +1217,81 @@ def generate_build_ninja(
order_only="post-build",
)
n.comment("Phony edge that will always be considered dirty by ninja.")
n.comment(
"This can be used as an implicit to a target that should always be rerun, ignoring file modified times."
)
n.build(
outputs="always",
rule="phony",
)
n.newline()
###
# Regression test progress reports
###
report_baseline_path = build_path / "baseline.json"
report_changes_path = build_path / "report_changes.json"
changes_fmt = config.tools_dir / "changes_fmt.py"
regressions_md = build_path / "regressions.md"
n.comment(
"Create a baseline progress report for later match regression testing"
)
n.build(
outputs=report_baseline_path,
rule="report",
implicit=[objdiff, "all_source", "always"],
order_only="post-build",
)
n.build(
outputs="baseline",
rule="phony",
inputs=report_baseline_path,
)
n.comment("Check for any match regressions against the baseline")
n.comment("Will fail if no baseline has been created")
n.rule(
name="report_changes",
command=f"{objdiff} report changes --format json-pretty {report_baseline_path} $in -o $out",
description="CHANGES",
)
n.build(
outputs=report_changes_path,
rule="report_changes",
inputs=report_path,
implicit=[objdiff, "always"],
)
n.rule(
name="changes_fmt",
command=f"$python {changes_fmt} $args $in",
description="CHANGESFMT",
)
n.build(
outputs="changes",
rule="changes_fmt",
inputs=report_changes_path,
implicit=changes_fmt,
)
n.build(
outputs="changes_all",
rule="changes_fmt",
inputs=report_changes_path,
implicit=changes_fmt,
variables={"args": "--all"},
)
n.rule(
name="changes_md",
command=f"$python {changes_fmt} $in -o $out",
description="CHANGESFMT $out",
)
n.build(
outputs=regressions_md,
rule="changes_md",
inputs=report_changes_path,
implicit=changes_fmt,
)
n.newline()
###
# Helper tools
###
@@ -1310,7 +1387,7 @@ def generate_build_ninja(
if config.non_matching:
n.default(link_outputs)
elif config.progress:
n.default(progress_path)
n.default("progress")
else:
n.default(ok_path)
else:
@@ -1369,6 +1446,7 @@ def generate_objdiff_config(
"GC/1.3.2": "mwcc_242_81",
"GC/1.3.2r": "mwcc_242_81r",
"GC/2.0": "mwcc_247_92",
"GC/2.0p1": "mwcc_247_92p1",
"GC/2.5": "mwcc_247_105",
"GC/2.6": "mwcc_247_107",
"GC/2.7": "mwcc_247_108",
@@ -1749,7 +1827,7 @@ def generate_compile_commands(
json.dump(clangd_config, w, indent=2, default=default_format)
# Calculate, print and write progress to progress.json
# Print progress information from objdiff report
def calculate_progress(config: ProjectConfig) -> None:
config.validate()
out_path = config.out_path()
@@ -1841,35 +1919,3 @@ def calculate_progress(config: ProjectConfig) -> None:
if summary_file:
summary_file.write("```\n")
summary_file.close()
# Generate and write progress.json
progress_json: Dict[str, Any] = {}
def add_category(id: str, measures: Dict[str, Any]) -> None:
progress_json[id] = {
"code": measures.get("complete_code", 0),
"code/total": measures.get("total_code", 0),
"data": measures.get("complete_data", 0),
"data/total": measures.get("total_data", 0),
"matched_code": measures.get("matched_code", 0),
"matched_code/total": measures.get("total_code", 0),
"matched_data": measures.get("matched_data", 0),
"matched_data/total": measures.get("total_data", 0),
"matched_functions": measures.get("matched_functions", 0),
"matched_functions/total": measures.get("total_functions", 0),
"fuzzy_match": int(measures.get("fuzzy_match_percent", 0) * 100),
"fuzzy_match/total": 10000,
"units": measures.get("complete_units", 0),
"units/total": measures.get("total_units", 0),
}
if config.progress_all:
add_category("all", report_data["measures"])
else:
# Support for old behavior where "dol" was the main category
add_category("dol", report_data["measures"])
for category in report_data.get("categories", []):
add_category(category["id"], category["measures"])
with open(out_path / "progress.json", "w", encoding="utf-8") as w:
json.dump(progress_json, w, indent=2)
tools/upload_progress.py
-79
View File
@@ -1,79 +0,0 @@
#!/usr/bin/env python3
###
# Uploads progress information to https://github.com/decompals/frogress.
#
# Usage:
# python3 tools/upload_progress.py -b https://progress.decomp.club/ -p [project] -v [version] build/[version]/progress.json
#
# If changes are made, please submit a PR to
# https://github.com/encounter/dtk-template
###
import argparse
import json
import os
import requests
import subprocess
import sys
def get_git_commit_timestamp() -> int:
return int(
subprocess.check_output(["git", "show", "-s", "--format=%ct"])
.decode("ascii")
.rstrip()
)
def get_git_commit_sha() -> str:
return subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("ascii").strip()
def generate_url(args: argparse.Namespace) -> str:
url_components = [args.base_url.rstrip("/"), "data"]
for arg in [args.project, args.version]:
if arg != "":
url_components.append(arg)
return str.join("/", url_components) + "/"
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Upload progress information.")
parser.add_argument("-b", "--base_url", help="API base URL", required=True)
parser.add_argument("-a", "--api_key", help="API key (env var PROGRESS_API_KEY)")
parser.add_argument("-p", "--project", help="Project slug", required=True)
parser.add_argument("-v", "--version", help="Version slug", required=True)
parser.add_argument("input", help="Progress JSON input")
args = parser.parse_args()
api_key = args.api_key or os.environ.get("PROGRESS_API_KEY")
if not api_key:
raise KeyError("API key required")
url = generate_url(args)
entries = []
with open(args.input, "r") as f:
data = json.load(f)
entries.append(
{
"timestamp": get_git_commit_timestamp(),
"git_hash": get_git_commit_sha(),
"categories": data,
}
)
print("Publishing entry to", url)
json.dump(entries[0], sys.stdout, indent=4)
print()
r = requests.post(
url,
json={
"api_key": api_key,
"entries": entries,
},
)
r.raise_for_status()
print("Done!")