diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4cab970de..2ebab4274 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,21 +40,7 @@ jobs: run: | python configure.py --map --version ${{ matrix.version }} \ --binutils /binutils --compilers /compilers - ninja diff - ninja all_source build/${{ matrix.version }}/progress.json \ - build/${{ matrix.version }}/report.json - - # Upload progress if we're on the main branch - - name: Upload progress - if: github.ref == 'refs/heads/main' - continue-on-error: true - env: - PROGRESS_SLUG: tww - PROGRESS_API_KEY: ${{ secrets.PROGRESS_API_KEY }} - run: | - python tools/upload_progress.py -b https://progress.decomp.club/ \ - -p $PROGRESS_SLUG -v ${{ matrix.version }} \ - build/${{ matrix.version }}/progress.json + ninja all_source progress build/${{ matrix.version }}/report.json # Upload map files - name: Upload map diff --git a/configure.py b/configure.py index a9ed35239..0d272e553 100755 --- a/configure.py +++ b/configure.py @@ -151,10 +151,10 @@ if args.no_asm: # Tool versions config.binutils_tag = "2.42-1" config.compilers_tag = "20240706" -config.dtk_tag = "v1.4.1" -config.objdiff_tag = "v3.0.0-beta.6" -config.sjiswrap_tag = "v1.2.0" -config.wibo_tag = "0.6.11" +config.dtk_tag = "v1.5.1" +config.objdiff_tag = "v3.0.0-beta.8" +config.sjiswrap_tag = "v1.2.1" +config.wibo_tag = "0.6.16" # Project config.config_path = Path("config") / config.version / "config.yml" @@ -1823,6 +1823,12 @@ config.progress_categories = [ ProgressCategory("third_party", "Third Party"), ] config.progress_each_module = args.verbose +# Optional extra arguments to `objdiff-cli report generate` +config.progress_report_args = [ + # Marks relocations as mismatching if the target value is different + # Default is "functionRelocDiffs=none", which is most lenient + # "--config functionRelocDiffs=data_value", +] # Disable missing return type warnings for incomplete objects for lib in config.libs: @@ -1834,7 +1840,7 @@ if args.mode == "configure": # Write build.ninja and objdiff.json generate_build(config) elif args.mode == "progress": - # Print progress and write progress.json + # Print progress information calculate_progress(config) else: sys.exit("Unknown mode: " + args.mode) diff --git a/tools/project.py b/tools/project.py index 6d65daa06..47e9f9d10 100644 --- a/tools/project.py +++ b/tools/project.py @@ -196,9 +196,8 @@ class ProjectConfig: None # Callback to add/remove/reorder units within a module ) - # Progress output, progress.json and report.json config + # Progress output and report.json config self.progress = True # Enable report.json generation and CLI progress output - self.progress_all: bool = True # Include combined "all" category self.progress_modules: bool = True # Include combined "modules" category self.progress_each_module: bool = ( False # Include individual modules, disable for large numbers of modules @@ -207,6 +206,9 @@ class ProjectConfig: self.print_progress_categories: Union[bool, List[str]] = ( True # Print additional progress categories in the CLI progress output ) + self.progress_report_args: Optional[List[str]] = ( + None # Flags to `objdiff-cli report generate` + ) # Progress fancy printing self.progress_use_fancy: bool = False @@ -423,6 +425,7 @@ def generate_build_ninja( if config.linker_version is None: sys.exit("ProjectConfig.linker_version missing") n.variable("mw_version", Path(config.linker_version)) + n.variable("objdiff_report_args", make_flags_str(config.progress_report_args)) n.newline() ### @@ -431,7 +434,6 @@ def generate_build_ninja( n.comment("Tooling") build_path = config.out_path() - progress_path = build_path / "progress.json" report_path = build_path / "report.json" build_tools_path = config.build_dir / "tools" download_tool = config.tools_dir / "download_tool.py" @@ -1188,7 +1190,7 @@ def generate_build_ninja( description="PROGRESS", ) n.build( - outputs=progress_path, + outputs="progress", rule="progress", implicit=[ ok_path, @@ -1205,7 +1207,7 @@ def generate_build_ninja( n.comment("Generate progress report") n.rule( name="report", - command=f"{objdiff} report generate -o $out", + command=f"{objdiff} report generate $objdiff_report_args -o $out", description="REPORT", ) n.build( @@ -1385,7 +1387,7 @@ def generate_build_ninja( if config.non_matching: n.default(link_outputs) elif config.progress: - n.default(progress_path) + n.default("progress") else: n.default(ok_path) else: @@ -1825,7 +1827,7 @@ def generate_compile_commands( json.dump(clangd_config, w, indent=2, default=default_format) -# Calculate, print and write progress to progress.json +# Print progress information from objdiff report def calculate_progress(config: ProjectConfig) -> None: config.validate() out_path = config.out_path() @@ -1917,35 +1919,3 @@ def calculate_progress(config: ProjectConfig) -> None: if summary_file: summary_file.write("```\n") summary_file.close() - - # Generate and write progress.json - progress_json: Dict[str, Any] = {} - - def add_category(id: str, measures: Dict[str, Any]) -> None: - progress_json[id] = { - "code": measures.get("complete_code", 0), - "code/total": measures.get("total_code", 0), - "data": measures.get("complete_data", 0), - "data/total": measures.get("total_data", 0), - "matched_code": measures.get("matched_code", 0), - "matched_code/total": measures.get("total_code", 0), - "matched_data": measures.get("matched_data", 0), - "matched_data/total": measures.get("total_data", 0), - "matched_functions": measures.get("matched_functions", 0), - "matched_functions/total": measures.get("total_functions", 0), - "fuzzy_match": int(measures.get("fuzzy_match_percent", 0) * 100), - "fuzzy_match/total": 10000, - "units": measures.get("complete_units", 0), - "units/total": measures.get("total_units", 0), - } - - if config.progress_all: - add_category("all", report_data["measures"]) - else: - # Support for old behavior where "dol" was the main category - add_category("dol", report_data["measures"]) - for category in report_data.get("categories", []): - add_category(category["id"], category["measures"]) - - with open(out_path / "progress.json", "w", encoding="utf-8") as w: - json.dump(progress_json, w, indent=2) diff --git a/tools/upload_progress.py b/tools/upload_progress.py deleted file mode 100755 index dc61d156e..000000000 --- a/tools/upload_progress.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 - -### -# Uploads progress information to https://github.com/decompals/frogress. -# -# Usage: -# python3 tools/upload_progress.py -b https://progress.decomp.club/ -p [project] -v [version] build/[version]/progress.json -# -# If changes are made, please submit a PR to -# https://github.com/encounter/dtk-template -### - -import argparse -import json -import os -import requests -import subprocess -import sys - - -def get_git_commit_timestamp() -> int: - return int( - subprocess.check_output(["git", "show", "-s", "--format=%ct"]) - .decode("ascii") - .rstrip() - ) - - -def get_git_commit_sha() -> str: - return subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("ascii").strip() - - -def generate_url(args: argparse.Namespace) -> str: - url_components = [args.base_url.rstrip("/"), "data"] - - for arg in [args.project, args.version]: - if arg != "": - url_components.append(arg) - - return str.join("/", url_components) + "/" - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Upload progress information.") - parser.add_argument("-b", "--base_url", help="API base URL", required=True) - parser.add_argument("-a", "--api_key", help="API key (env var PROGRESS_API_KEY)") - parser.add_argument("-p", "--project", help="Project slug", required=True) - parser.add_argument("-v", "--version", help="Version slug", required=True) - parser.add_argument("input", help="Progress JSON input") - - args = parser.parse_args() - api_key = args.api_key or os.environ.get("PROGRESS_API_KEY") - if not api_key: - raise KeyError("API key required") - url = generate_url(args) - - entries = [] - with open(args.input, "r") as f: - data = json.load(f) - entries.append( - { - "timestamp": get_git_commit_timestamp(), - "git_hash": get_git_commit_sha(), - "categories": data, - } - ) - - print("Publishing entry to", url) - json.dump(entries[0], sys.stdout, indent=4) - print() - r = requests.post( - url, - json={ - "api_key": api_key, - "entries": entries, - }, - ) - r.raise_for_status() - print("Done!")