Update dtk-template (#2453)

* Update dtk-template

* Fix debug build

* Fix d_event_debug includes

* Progress reports: Enable function relocation diffing (data_value)

* Fix some diffs

* Fix a few more diffs

* More fixes

tools/changes_fmt.py

@@ -4,7 +4,7 @@ from argparse import ArgumentParser
 import os
 import json
 from pathlib import Path
-from typing import Tuple
+from typing import Optional, Tuple
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.abspath(os.path.join(script_dir, ".."))
@@ -22,10 +22,16 @@ FUNCTION_KEYS_TO_DIFF = [
     "fuzzy_match_percent",
 ]
 
-type Change = Tuple[str, str, float, float]
+Change = Tuple[str, str, float, float]
 
 
-def get_changes(changes_file: str) -> list[Change]:
+def format_float(value: float) -> str:
+    if value < 100.0 and value > 99.99:
+        value = 99.99
+    return "%6.2f" % value
+
+
+def get_changes(changes_file: str) -> Tuple[list[Change], list[Change]]:
     changes_file = os.path.relpath(changes_file, root_dir)
     with open(changes_file, "r") as f:
         changes_json = json.load(f)
@@ -33,7 +39,7 @@ def get_changes(changes_file: str) -> list[Change]:
     regressions = []
     progressions = []
 
-    def diff_key(object_name: str, object: dict, key: str):
+    def diff_key(object_name: Optional[str], object: dict, key: str):
         from_value = object.get("from", {}).get(key, 0.0)
         to_value = object.get("to", {}).get(key, 0.0)
         key = key.removesuffix("_percent")
@@ -77,7 +83,7 @@ def generate_changes_plaintext(changes: list[Change]) -> str:
         if len(name) > name_max_len:
             name = name[: name_max_len - len("[...]")] + "[...]"
         out_lines.append(
-            f"{name:>{name_max_len}} | {key:<{key_max_len}} | {from_value:6.2f}% -> {to_value:5.2f}%"
+            f"{name:>{name_max_len}} | {key:<{key_max_len}} | {format_float(from_value)}% -> {format_float(to_value)}%"
         )
 
     return "\n".join(out_lines)
@@ -106,7 +112,9 @@ def generate_changes_markdown(changes: list[Change], description: str) -> str:
                 name = name[: name_max_len - len("...")] + "..."
             name = f"`{name}`"  # Surround with backticks
         key = key.replace("_", " ").capitalize()
-        out_lines.append(f"| {name} | {key} | {from_value:.2f}% | {to_value:.2f}% |")
+        out_lines.append(
+            f"| {name} | {key} | {format_float(from_value)}% | {format_float(to_value)}% |"
+        )
 
     out_lines.append("</details>")
 

tools/project.py

@@ -196,9 +196,8 @@ class ProjectConfig:
             None  # Callback to add/remove/reorder units within a module
         )
 
-        # Progress output, progress.json and report.json config
+        # Progress output and report.json config
         self.progress = True  # Enable report.json generation and CLI progress output
-        self.progress_all: bool = True  # Include combined "all" category
         self.progress_modules: bool = True  # Include combined "modules" category
         self.progress_each_module: bool = (
             False  # Include individual modules, disable for large numbers of modules
@@ -207,6 +206,9 @@ class ProjectConfig:
         self.print_progress_categories: Union[bool, List[str]] = (
             True  # Print additional progress categories in the CLI progress output
         )
+        self.progress_report_args: Optional[List[str]] = (
+            None  # Flags to `objdiff-cli report generate`
+        )
 
         # Progress fancy printing
         self.progress_use_fancy: bool = False
@@ -423,6 +425,7 @@ def generate_build_ninja(
     if config.linker_version is None:
         sys.exit("ProjectConfig.linker_version missing")
     n.variable("mw_version", Path(config.linker_version))
+    n.variable("objdiff_report_args", make_flags_str(config.progress_report_args))
     n.newline()
 
     ###
@@ -431,7 +434,6 @@ def generate_build_ninja(
     n.comment("Tooling")
 
     build_path = config.out_path()
-    progress_path = build_path / "progress.json"
     report_path = build_path / "report.json"
     build_tools_path = config.build_dir / "tools"
     download_tool = config.tools_dir / "download_tool.py"
@@ -1188,7 +1190,7 @@ def generate_build_ninja(
             description="PROGRESS",
         )
         n.build(
-            outputs=progress_path,
+            outputs="progress",
             rule="progress",
             implicit=[
                 ok_path,
@@ -1205,7 +1207,7 @@ def generate_build_ninja(
         n.comment("Generate progress report")
         n.rule(
             name="report",
-            command=f"{objdiff} report generate -o $out",
+            command=f"{objdiff} report generate $objdiff_report_args -o $out",
             description="REPORT",
         )
         n.build(
@@ -1385,7 +1387,7 @@ def generate_build_ninja(
         if config.non_matching:
             n.default(link_outputs)
         elif config.progress:
-            n.default(progress_path)
+            n.default("progress")
         else:
             n.default(ok_path)
     else:
@@ -1437,6 +1439,7 @@ def generate_objdiff_config(
     COMPILER_MAP = {
         "GC/1.0": "mwcc_233_144",
         "GC/1.1": "mwcc_233_159",
+        "GC/1.1p1": "mwcc_233_159p1",
         "GC/1.2.5": "mwcc_233_163",
         "GC/1.2.5e": "mwcc_233_163e",
         "GC/1.2.5n": "mwcc_233_163n",
@@ -1825,7 +1828,7 @@ def generate_compile_commands(
         json.dump(clangd_config, w, indent=2, default=default_format)
 
 
-# Calculate, print and write progress to progress.json
+# Print progress information from objdiff report
 def calculate_progress(config: ProjectConfig) -> None:
     config.validate()
     out_path = config.out_path()
@@ -1917,35 +1920,3 @@ def calculate_progress(config: ProjectConfig) -> None:
     if summary_file:
         summary_file.write("```\n")
         summary_file.close()
-
-    # Generate and write progress.json
-    progress_json: Dict[str, Any] = {}
-
-    def add_category(id: str, measures: Dict[str, Any]) -> None:
-        progress_json[id] = {
-            "code": measures.get("complete_code", 0),
-            "code/total": measures.get("total_code", 0),
-            "data": measures.get("complete_data", 0),
-            "data/total": measures.get("total_data", 0),
-            "matched_code": measures.get("matched_code", 0),
-            "matched_code/total": measures.get("total_code", 0),
-            "matched_data": measures.get("matched_data", 0),
-            "matched_data/total": measures.get("total_data", 0),
-            "matched_functions": measures.get("matched_functions", 0),
-            "matched_functions/total": measures.get("total_functions", 0),
-            "fuzzy_match": int(measures.get("fuzzy_match_percent", 0) * 100),
-            "fuzzy_match/total": 10000,
-            "units": measures.get("complete_units", 0),
-            "units/total": measures.get("total_units", 0),
-        }
-
-    if config.progress_all:
-        add_category("all", report_data["measures"])
-    else:
-        # Support for old behavior where "dol" was the main category
-        add_category("dol", report_data["measures"])
-    for category in report_data.get("categories", []):
-        add_category(category["id"], category["measures"])
-
-    with open(out_path / "progress.json", "w", encoding="utf-8") as w:
-        json.dump(progress_json, w, indent=2)

tools/upload_progress.py

@@ -1,79 +0,0 @@
-#!/usr/bin/env python3
-
-###
-# Uploads progress information to https://github.com/decompals/frogress.
-#
-# Usage:
-#   python3 tools/upload_progress.py -b https://progress.decomp.club/ -p [project] -v [version] build/[version]/progress.json
-#
-# If changes are made, please submit a PR to
-# https://github.com/encounter/dtk-template
-###
-
-import argparse
-import json
-import os
-import requests
-import subprocess
-import sys
-
-
-def get_git_commit_timestamp() -> int:
-    return int(
-        subprocess.check_output(["git", "show", "-s", "--format=%ct"])
-        .decode("ascii")
-        .rstrip()
-    )
-
-
-def get_git_commit_sha() -> str:
-    return subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("ascii").strip()
-
-
-def generate_url(args: argparse.Namespace) -> str:
-    url_components = [args.base_url.rstrip("/"), "data"]
-
-    for arg in [args.project, args.version]:
-        if arg != "":
-            url_components.append(arg)
-
-    return str.join("/", url_components) + "/"
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Upload progress information.")
-    parser.add_argument("-b", "--base_url", help="API base URL", required=True)
-    parser.add_argument("-a", "--api_key", help="API key (env var PROGRESS_API_KEY)")
-    parser.add_argument("-p", "--project", help="Project slug", required=True)
-    parser.add_argument("-v", "--version", help="Version slug", required=True)
-    parser.add_argument("input", help="Progress JSON input")
-
-    args = parser.parse_args()
-    api_key = args.api_key or os.environ.get("PROGRESS_API_KEY")
-    if not api_key:
-        raise KeyError("API key required")
-    url = generate_url(args)
-
-    entries = []
-    with open(args.input, "r") as f:
-        data = json.load(f)
-        entries.append(
-            {
-                "timestamp": get_git_commit_timestamp(),
-                "git_hash": get_git_commit_sha(),
-                "categories": data,
-            }
-        )
-
-    print("Publishing entry to", url)
-    json.dump(entries[0], sys.stdout, indent=4)
-    print()
-    r = requests.post(
-        url,
-        json={
-            "api_key": api_key,
-            "entries": entries,
-        },
-    )
-    r.raise_for_status()
-    print("Done!")