Add progress calculation & upload to frogress

.github/workflows/build.yml

@@ -23,6 +23,14 @@ jobs:
       run: |
         python configure.py --map --version ${{matrix.version}} --compilers /compilers
         ninja
+    - name: Upload progress
+      if: github.ref == 'refs/heads/main'
+      continue-on-error: true
+      env:
+        PROGRESS_API_KEY: ${{secrets.PROGRESS_API_KEY}}
+      run: |
+        python tools/upload_progress.py -b https://progress.deco.mp/ -p tww -v ${{matrix.version}} \
+          build/${{matrix.version}}/progress.json
     - name: Upload maps
       uses: actions/upload-artifact@v3
       with:

README.md

@@ -1,13 +1,17 @@
-The Legend of Zelda: The Wind Waker [![Build Status]][actions]
+The Legend of Zelda: The Wind Waker  
+[![Build Status]][actions] ![Progress] ![DOL Progress] ![RELs Progress]
 =============
 
 [Build Status]: https://github.com/zeldaret/tww/actions/workflows/build.yml/badge.svg
 [actions]: https://github.com/zeldaret/tww/actions/workflows/build.yml
+[Progress]: https://img.shields.io/endpoint?label=Code&url=https%3A%2F%2Fprogress.deco.mp%2Fdata%2Ftww%2FGZLE01%2Fall%2F%3Fmode%3Dshield%26measure%3Dcode
+[DOL Progress]: https://img.shields.io/endpoint?label=DOL&url=https%3A%2F%2Fprogress.deco.mp%2Fdata%2Ftww%2FGZLE01%2Fdol%2F%3Fmode%3Dshield%26measure%3Dcode
+[RELs Progress]: https://img.shields.io/endpoint?label=RELs&url=https%3A%2F%2Fprogress.deco.mp%2Fdata%2Ftww%2FGZLE01%2Fmodules%2F%3Fmode%3Dshield%26measure%3Dcode
 
 This repository supports the following versions:
 
-- GZLE01 - Rev 0 (USA), Rev 48 (Korea)
-- GZLP01 - Rev 0 (EUR)
+- GZLE01 - Rev 0 (USA), Rev 48 (KOR)
+- GZLP01 - Rev 0 (PAL)
 - GZLJ01 - Rev 0 (JPN)
 
 Dependencies
@@ -43,11 +47,11 @@ Linux:
 Building
 ========
 
-- Checkout the repository:
+- Clone the repository:
   ```
   git clone https://github.com/zeldaret/tww.git
   ```
-- Using [Dolphin Emulator](https://dolphin-emu.org/), extract your game to `orig/GZLE01` (or `GLZJ01` for JPN, `GLZE01` for PAL).  
+- Using [Dolphin Emulator](https://dolphin-emu.org/), extract your game to `orig/GZLE01` (or `GZLJ01` for JPN, `GZLE01` for PAL).  
 ![](assets/dolphin-extract.png)
   - To save space, the only necessary files are the following. Any others can be deleted.
     - `sys/main.dol`
@@ -57,6 +61,7 @@ Building
   ```
   python configure.py
   ```
+  To use a version other than `GZLE01` (USA), specify `--version GZLJ01` (JPN) or `--version GZLP01` (PAL).
 - Build:
   ```
   ninja

configure.py

@@ -19,6 +19,7 @@ from pathlib import Path
 from tools.project import (
     Object,
     ProjectConfig,
+    calculate_progress,
     generate_build,
     is_windows,
 )
@@ -37,6 +38,12 @@ else:
     versions_str = VERSIONS[0]
 
 parser = argparse.ArgumentParser()
+parser.add_argument(
+    "mode",
+    default="configure",
+    help="configure or progress (default: configure)",
+    nargs='?',
+)
 parser.add_argument(
     "--version",
     dest="version",
@@ -87,6 +94,12 @@ parser.add_argument(
     type=Path,
     help="path to sjiswrap.exe (optional)",
 )
+parser.add_argument(
+    "--verbose",
+    dest="verbose",
+    action="store_true",
+    help="print verbose output",
+)
 args = parser.parse_args()
 
 config = ProjectConfig()
@@ -107,7 +120,7 @@ if not is_windows():
 
 # Tool versions
 config.compilers_tag = "1"
-config.dtk_tag = "v0.5.1"
+config.dtk_tag = "v0.5.2"
 config.sjiswrap_tag = "v1.1.0"
 config.wibo_tag = "0.5.1"
 
@@ -796,5 +809,12 @@ config.libs = [
     ActorRel(NonMatching, "d_a_movie_player"),
 ]
 
-# Write build.ninja and objdiff.json
-generate_build(config)
+if args.mode == "configure":
+    # Write build.ninja and objdiff.json
+    generate_build(config)
+elif args.mode == "progress":
+    # Print progress and write progress.json
+    config.progress_each_module = args.verbose
+    calculate_progress(config)
+else:
+    sys.exit("Unknown mode: " + args.mode)

tools/project.py

@@ -28,31 +28,37 @@ if sys.platform == "cygwin":
 
 
 class ProjectConfig:
-    # Paths
-    build_dir = Path("build")
-    src_dir = Path("src")
-    tools_dir = Path("tools")
+    def __init__(self):
+        # Paths
+        self.build_dir = Path("build")
+        self.src_dir = Path("src")
+        self.tools_dir = Path("tools")
 
-    # Tooling
-    dtk_tag = None  # Git tag
-    build_dtk_path = None  # If None, download
-    compilers_tag = None  # 1
-    compilers_path = None  # If None, download
-    wibo_tag = None  # Git tag
-    wrapper = None  # If None, download wibo on Linux
-    sjiswrap_tag = None  # Git tag
-    sjiswrap_path = None  # If None, download
+        # Tooling
+        self.dtk_tag = None  # Git tag
+        self.build_dtk_path = None  # If None, download
+        self.compilers_tag = None  # 1
+        self.compilers_path = None  # If None, download
+        self.wibo_tag = None  # Git tag
+        self.wrapper = None  # If None, download wibo on Linux
+        self.sjiswrap_tag = None  # Git tag
+        self.sjiswrap_path = None  # If None, download
 
-    # Project config
-    check_sha_path = None  # Path to version.sha1
-    config_path = None  # Path to config.yml
-    build_rels = True  # Build REL files
-    debug = False  # Build with debug info
-    generate_map = False  # Generate map file(s)
-    ldflags = None  # Linker flags
-    linker_version = None  # mwld version
-    libs = None  # List of libraries
-    version = None  # Version name
+        # Project config
+        self.check_sha_path = None  # Path to version.sha1
+        self.config_path = None  # Path to config.yml
+        self.build_rels = True  # Build REL files
+        self.debug = False  # Build with debug info
+        self.generate_map = False  # Generate map file(s)
+        self.ldflags = None  # Linker flags
+        self.linker_version = None  # mwld version
+        self.libs = None  # List of libraries
+        self.version = None  # Version name
+
+        # Progress output and progress.json config
+        self.progress_all = True  # Include combined "all" category
+        self.progress_modules = True  # Include combined "modules" category
+        self.progress_each_module = True  # Include individual modules, disable for large numbers of modules
 
     def validate(self):
         required_attrs = [
@@ -127,20 +133,30 @@ def path(value):
 
 # Load decomp-toolkit generated config.json
 def load_build_config(config, build_config_path):
+    if not build_config_path.is_file():
+        return None
+
     def versiontuple(v):
         return tuple(map(int, (v.split("."))))
 
-    if build_config_path.is_file():
-        with open(build_config_path) as r:
-            build_config = json.load(r)
-            config_version = build_config.get("version")
-            if not config_version:
-                return None
-            dtk_version = config.dtk_tag[1:]  # Strip v
-            if versiontuple(config_version) < versiontuple(dtk_version):
-                return None
-            return build_config
-    return None
+    f = open(build_config_path, "r", encoding="utf-8")
+    build_config = json.load(f)
+    config_version = build_config.get("version")
+    if not config_version:
+        # Invalid config.json
+        f.close()
+        os.remove(build_config_path)
+        return None
+
+    dtk_version = config.dtk_tag[1:]  # Strip v
+    if versiontuple(config_version) < versiontuple(dtk_version):
+        # Outdated config.json
+        f.close()
+        os.remove(build_config_path)
+        return None
+
+    f.close()
+    return build_config
 
 
 # Generate build.ninja and objdiff.json
@@ -159,6 +175,9 @@ def generate_build_ninja(config, build_config):
     n.variable("ninja_required_version", "1.3")
     n.newline()
 
+    configure_script = os.path.relpath(os.path.abspath(sys.argv[0]))
+    python_lib = os.path.relpath(__file__)
+    python_lib_dir = os.path.dirname(python_lib)
     n.comment("The arguments passed to configure.py, for rerunning it.")
     n.variable("configure_args", sys.argv[1:])
     n.variable("python", f'"{sys.executable}"')
@@ -204,6 +223,7 @@ def generate_build_ninja(config, build_config):
             outputs=path(dtk),
             rule="cargo",
             inputs=path(config.build_dtk_path / "Cargo.toml"),
+            implicit=path(config.build_dtk_path / "Cargo.lock"),
             variables={
                 "bin": "dtk",
                 "target": build_tools_path,
@@ -634,6 +654,22 @@ def generate_build_ninja(config, build_config):
         )
         n.newline()
 
+        ###
+        # Calculate progress
+        ###
+        n.comment("Calculate progress")
+        progress_path = build_path / "progress.json"
+        n.rule(
+            name="progress",
+            command=f"$python {configure_script} $configure_args progress",
+            description="PROGRESS",
+        )
+        n.build(
+            outputs=path(progress_path),
+            rule="progress",
+            implicit=path([ok_path, configure_script, python_lib, config.config_path]),
+        )
+
         ###
         # Helper tools
         ###
@@ -701,14 +737,11 @@ def generate_build_ninja(config, build_config):
     # Regenerate on change
     ###
     n.comment("Reconfigure on change")
-    python_script = os.path.relpath(os.path.abspath(sys.argv[0]))
-    python_lib = os.path.relpath(__file__)
-    python_lib_dir = os.path.dirname(python_lib)
     n.rule(
         name="configure",
-        command=f"$python {python_script} $configure_args",
+        command=f"$python {configure_script} $configure_args",
         generator=True,
-        description=f"RUN {python_script}",
+        description=f"RUN {configure_script}",
     )
     n.build(
         outputs="build.ninja",
@@ -716,7 +749,7 @@ def generate_build_ninja(config, build_config):
         implicit=path(
             [
                 build_config_path,
-                python_script,
+                configure_script,
                 python_lib,
                 Path(python_lib_dir) / "ninja_syntax.py",
             ]
@@ -729,7 +762,7 @@ def generate_build_ninja(config, build_config):
     ###
     n.comment("Default rule")
     if build_config:
-        n.default(path(ok_path))
+        n.default(path(progress_path))
     else:
         n.default(path(build_config_path))
 
@@ -818,3 +851,121 @@ def generate_objdiff_config(config, build_config):
     # Write objdiff.json
     with open("objdiff.json", "w", encoding="utf-8") as w:
         json.dump(objdiff_config, w, indent=4)
+
+
+# Calculate, print and write progress to progress.json
+def calculate_progress(config):
+    out_path = config.out_path()
+    build_config = load_build_config(config, out_path / "config.json")
+    if not build_config:
+        return
+
+    class ProgressUnit:
+        def __init__(self, name):
+            self.name = name
+            self.code_total = 0
+            self.code_progress = 0
+            self.data_total = 0
+            self.data_progress = 0
+            self.objects_progress = 0
+            self.objects_total = 0
+            self.objects = set()
+
+        def add(self, build_obj):
+            self.code_total += build_obj["code_size"]
+            self.data_total += build_obj["data_size"]
+
+            # Avoid counting the same object in different modules twice
+            include_object = build_obj["name"] not in self.objects
+            if include_object:
+                self.objects.add(build_obj["name"])
+                self.objects_total += 1
+
+            if build_obj["autogenerated"]:
+                # Skip autogenerated objects
+                return
+
+            result = config.find_object(build_obj["name"])
+            if not result:
+                return
+
+            _, obj = result
+            if not obj.completed:
+                return
+
+            self.code_progress += build_obj["code_size"]
+            self.data_progress += build_obj["data_size"]
+            if include_object:
+                self.objects_progress += 1
+
+        def code_frac(self):
+            return self.code_progress / self.code_total
+
+        def data_frac(self):
+            return self.data_progress / self.data_total
+
+    # Add DOL units
+    all_progress = ProgressUnit("All") if config.progress_all else None
+    dol_progress = ProgressUnit("DOL")
+    for unit in build_config["units"]:
+        if all_progress:
+            all_progress.add(unit)
+        dol_progress.add(unit)
+
+    # Add REL units
+    rels_progress = ProgressUnit("Modules") if config.progress_modules else None
+    modules_progress = []
+    for module in build_config["modules"]:
+        progress = ProgressUnit(module["name"])
+        modules_progress.append(progress)
+        for unit in module["units"]:
+            if all_progress:
+                all_progress.add(unit)
+            if rels_progress:
+                rels_progress.add(unit)
+            progress.add(unit)
+
+    # Print human-readable progress
+    print("Progress:")
+
+    def print_category(unit):
+        code_frac = unit.code_frac()
+        data_frac = unit.data_frac()
+        print(
+            f"  {unit.name}: {code_frac:.2%} code, {data_frac:.2%} data ({unit.objects_progress} / {unit.objects_total} files)"
+        )
+        print(f"    Code: {unit.code_progress} / {unit.code_total} bytes")
+        print(f"    Data: {unit.data_progress} / {unit.data_total} bytes")
+
+    if all_progress:
+        print_category(all_progress)
+    print_category(dol_progress)
+    module_count = len(build_config["modules"])
+    if module_count > 0:
+        print_category(rels_progress)
+        if config.progress_each_module:
+            for progress in modules_progress:
+                print_category(progress)
+
+    # Generate and write progress.json
+    progress_json = {}
+
+    def add_category(category, unit):
+        progress_json[category] = {
+            "code": unit.code_progress,
+            "code/total": unit.code_total,
+            "data": unit.data_progress,
+            "data/total": unit.data_total,
+        }
+
+    if all_progress:
+        add_category("all", all_progress)
+    add_category("dol", dol_progress)
+    if len(build_config["modules"]) > 0:
+        if rels_progress:
+            add_category("modules", rels_progress)
+        if config.progress_each_module:
+            for progress in modules_progress:
+                add_category(progress.name, progress)
+    with open(out_path / "progress.json", "w", encoding="utf-8") as w:
+        json.dump(progress_json, w, indent=4)

tools/upload_progress.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+###
+# Uploads progress information to https://github.com/decompals/frogress.
+#
+# Usage:
+#   python3 tools/upload_progress.py -b https://progress.deco.mp/ -p [project] -v [version] build/[version]/progress.json
+#
+# If changes are made, please submit a PR to
+# https://github.com/encounter/dtk-template
+###
+
+import argparse
+import json
+import os
+import requests
+import subprocess
+import sys
+
+
+def get_git_commit_timestamp() -> int:
+    return int(
+        subprocess.check_output(["git", "show", "-s", "--format=%ct"])
+        .decode("ascii")
+        .rstrip()
+    )
+
+
+def get_git_commit_sha() -> str:
+    return subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("ascii").strip()
+
+
+def generate_url(args: argparse.Namespace) -> str:
+    url_components = [args.base_url.rstrip("/"), "data"]
+
+    for arg in [args.project, args.version]:
+        if arg != "":
+            url_components.append(arg)
+
+    return str.join("/", url_components) + "/"
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Upload progress information.")
+    parser.add_argument("-b", "--base_url", help="API base URL", required=True)
+    parser.add_argument("-a", "--api_key", help="API key (env var PROGRESS_API_KEY)")
+    parser.add_argument("-p", "--project", help="Project slug", required=True)
+    parser.add_argument("-v", "--version", help="Version slug", required=True)
+    parser.add_argument("input", help="Progress JSON input")
+
+    args = parser.parse_args()
+    api_key = args.api_key or os.environ.get("PROGRESS_API_KEY")
+    if not api_key:
+        raise "API key required"
+    url = generate_url(args)
+
+    entries = []
+    with open(args.input, "r") as f:
+        data = json.load(f)
+        entries.append(
+            {
+                "timestamp": get_git_commit_timestamp(),
+                "git_hash": get_git_commit_sha(),
+                "categories": data,
+            }
+        )
+
+    print("Publishing entry to", url)
+    json.dump(entries[0], sys.stdout, indent=4)
+    print()
+    r = requests.post(url, json={
+        "api_key": api_key,
+        "entries": entries,
+    })
+    r.raise_for_status()
+    print("Done!")