Update dtk-template

.github/workflows/build.yml

@@ -40,21 +40,7 @@ jobs:
       run: |
         python configure.py --map --version ${{ matrix.version }} \
             --binutils /binutils --compilers /compilers
-        ninja diff
-        ninja all_source build/${{ matrix.version }}/progress.json \
-            build/${{ matrix.version }}/report.json
-
-    # Upload progress if we're on the main branch
-    - name: Upload progress
-      if: github.ref == 'refs/heads/main'
-      continue-on-error: true
-      env:
-        PROGRESS_SLUG: tww
-        PROGRESS_API_KEY: ${{ secrets.PROGRESS_API_KEY }}
-      run: |
-        python tools/upload_progress.py -b https://progress.decomp.club/ \
-          -p $PROGRESS_SLUG -v ${{ matrix.version }} \
-          build/${{ matrix.version }}/progress.json
+        ninja all_source progress build/${{ matrix.version }}/report.json
 
     # Upload map files
     - name: Upload map

configure.py

@@ -151,10 +151,10 @@ if args.no_asm:
 # Tool versions
 config.binutils_tag = "2.42-1"
 config.compilers_tag = "20240706"
-config.dtk_tag = "v1.4.1"
-config.objdiff_tag = "v3.0.0-beta.6"
-config.sjiswrap_tag = "v1.2.0"
-config.wibo_tag = "0.6.11"
+config.dtk_tag = "v1.5.1"
+config.objdiff_tag = "v3.0.0-beta.8"
+config.sjiswrap_tag = "v1.2.1"
+config.wibo_tag = "0.6.16"
 
 # Project
 config.config_path = Path("config") / config.version / "config.yml"
@@ -1823,6 +1823,12 @@ config.progress_categories = [
     ProgressCategory("third_party", "Third Party"),
 ]
 config.progress_each_module = args.verbose
+# Optional extra arguments to `objdiff-cli report generate`
+config.progress_report_args = [
+    # Marks relocations as mismatching if the target value is different
+    # Default is "functionRelocDiffs=none", which is most lenient
+    # "--config functionRelocDiffs=data_value",
+]
 
 # Disable missing return type warnings for incomplete objects
 for lib in config.libs:
@@ -1834,7 +1840,7 @@ if args.mode == "configure":
     # Write build.ninja and objdiff.json
     generate_build(config)
 elif args.mode == "progress":
-    # Print progress and write progress.json
+    # Print progress information
     calculate_progress(config)
 else:
     sys.exit("Unknown mode: " + args.mode)

tools/project.py

@@ -196,9 +196,8 @@ class ProjectConfig:
             None  # Callback to add/remove/reorder units within a module
         )
 
-        # Progress output, progress.json and report.json config
+        # Progress output and report.json config
         self.progress = True  # Enable report.json generation and CLI progress output
-        self.progress_all: bool = True  # Include combined "all" category
         self.progress_modules: bool = True  # Include combined "modules" category
         self.progress_each_module: bool = (
             False  # Include individual modules, disable for large numbers of modules
@@ -207,6 +206,9 @@ class ProjectConfig:
         self.print_progress_categories: Union[bool, List[str]] = (
             True  # Print additional progress categories in the CLI progress output
         )
+        self.progress_report_args: Optional[List[str]] = (
+            None  # Flags to `objdiff-cli report generate`
+        )
 
         # Progress fancy printing
         self.progress_use_fancy: bool = False
@@ -423,6 +425,7 @@ def generate_build_ninja(
     if config.linker_version is None:
         sys.exit("ProjectConfig.linker_version missing")
     n.variable("mw_version", Path(config.linker_version))
+    n.variable("objdiff_report_args", make_flags_str(config.progress_report_args))
     n.newline()
 
     ###
@@ -431,7 +434,6 @@ def generate_build_ninja(
     n.comment("Tooling")
 
     build_path = config.out_path()
-    progress_path = build_path / "progress.json"
     report_path = build_path / "report.json"
     build_tools_path = config.build_dir / "tools"
     download_tool = config.tools_dir / "download_tool.py"
@@ -1188,7 +1190,7 @@ def generate_build_ninja(
             description="PROGRESS",
         )
         n.build(
-            outputs=progress_path,
+            outputs="progress",
             rule="progress",
             implicit=[
                 ok_path,
@@ -1205,7 +1207,7 @@ def generate_build_ninja(
         n.comment("Generate progress report")
         n.rule(
             name="report",
-            command=f"{objdiff} report generate -o $out",
+            command=f"{objdiff} report generate $objdiff_report_args -o $out",
             description="REPORT",
         )
         n.build(
@@ -1385,7 +1387,7 @@ def generate_build_ninja(
         if config.non_matching:
             n.default(link_outputs)
         elif config.progress:
-            n.default(progress_path)
+            n.default("progress")
         else:
             n.default(ok_path)
     else:
@@ -1825,7 +1827,7 @@ def generate_compile_commands(
         json.dump(clangd_config, w, indent=2, default=default_format)
 
 
-# Calculate, print and write progress to progress.json
+# Print progress information from objdiff report
 def calculate_progress(config: ProjectConfig) -> None:
     config.validate()
     out_path = config.out_path()
@@ -1917,35 +1919,3 @@ def calculate_progress(config: ProjectConfig) -> None:
     if summary_file:
         summary_file.write("```\n")
         summary_file.close()
-
-    # Generate and write progress.json
-    progress_json: Dict[str, Any] = {}
-
-    def add_category(id: str, measures: Dict[str, Any]) -> None:
-        progress_json[id] = {
-            "code": measures.get("complete_code", 0),
-            "code/total": measures.get("total_code", 0),
-            "data": measures.get("complete_data", 0),
-            "data/total": measures.get("total_data", 0),
-            "matched_code": measures.get("matched_code", 0),
-            "matched_code/total": measures.get("total_code", 0),
-            "matched_data": measures.get("matched_data", 0),
-            "matched_data/total": measures.get("total_data", 0),
-            "matched_functions": measures.get("matched_functions", 0),
-            "matched_functions/total": measures.get("total_functions", 0),
-            "fuzzy_match": int(measures.get("fuzzy_match_percent", 0) * 100),
-            "fuzzy_match/total": 10000,
-            "units": measures.get("complete_units", 0),
-            "units/total": measures.get("total_units", 0),
-        }
-
-    if config.progress_all:
-        add_category("all", report_data["measures"])
-    else:
-        # Support for old behavior where "dol" was the main category
-        add_category("dol", report_data["measures"])
-    for category in report_data.get("categories", []):
-        add_category(category["id"], category["measures"])
-
-    with open(out_path / "progress.json", "w", encoding="utf-8") as w:
-        json.dump(progress_json, w, indent=2)

tools/upload_progress.py

@@ -1,79 +0,0 @@
-#!/usr/bin/env python3
-
-###
-# Uploads progress information to https://github.com/decompals/frogress.
-#
-# Usage:
-#   python3 tools/upload_progress.py -b https://progress.decomp.club/ -p [project] -v [version] build/[version]/progress.json
-#
-# If changes are made, please submit a PR to
-# https://github.com/encounter/dtk-template
-###
-
-import argparse
-import json
-import os
-import requests
-import subprocess
-import sys
-
-
-def get_git_commit_timestamp() -> int:
-    return int(
-        subprocess.check_output(["git", "show", "-s", "--format=%ct"])
-        .decode("ascii")
-        .rstrip()
-    )
-
-
-def get_git_commit_sha() -> str:
-    return subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("ascii").strip()
-
-
-def generate_url(args: argparse.Namespace) -> str:
-    url_components = [args.base_url.rstrip("/"), "data"]
-
-    for arg in [args.project, args.version]:
-        if arg != "":
-            url_components.append(arg)
-
-    return str.join("/", url_components) + "/"
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Upload progress information.")
-    parser.add_argument("-b", "--base_url", help="API base URL", required=True)
-    parser.add_argument("-a", "--api_key", help="API key (env var PROGRESS_API_KEY)")
-    parser.add_argument("-p", "--project", help="Project slug", required=True)
-    parser.add_argument("-v", "--version", help="Version slug", required=True)
-    parser.add_argument("input", help="Progress JSON input")
-
-    args = parser.parse_args()
-    api_key = args.api_key or os.environ.get("PROGRESS_API_KEY")
-    if not api_key:
-        raise KeyError("API key required")
-    url = generate_url(args)
-
-    entries = []
-    with open(args.input, "r") as f:
-        data = json.load(f)
-        entries.append(
-            {
-                "timestamp": get_git_commit_timestamp(),
-                "git_hash": get_git_commit_sha(),
-                "categories": data,
-            }
-        )
-
-    print("Publishing entry to", url)
-    json.dump(entries[0], sys.stdout, indent=4)
-    print()
-    r = requests.post(
-        url,
-        json={
-            "api_key": api_key,
-            "entries": entries,
-        },
-    )
-    r.raise_for_status()
-    print("Done!")