From d52641ed3fea645e87ee49fa34815e9b289a254e Mon Sep 17 00:00:00 2001 From: LagoLunatic Date: Tue, 15 Apr 2025 20:09:44 -0400 Subject: [PATCH] Add `ninja changes` command for regression testing --- tools/changes_fmt.py | 154 +++++++++++++++++++++++++++++++++++++++++++ tools/project.py | 75 +++++++++++++++++++++ 2 files changed, 229 insertions(+) create mode 100755 tools/changes_fmt.py diff --git a/tools/changes_fmt.py b/tools/changes_fmt.py new file mode 100755 index 000000000..49b997d3f --- /dev/null +++ b/tools/changes_fmt.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 + +from argparse import ArgumentParser +import os +import json +from pathlib import Path +from typing import Tuple + +script_dir = os.path.dirname(os.path.realpath(__file__)) +root_dir = os.path.abspath(os.path.join(script_dir, "..")) + + +UNIT_KEYS_TO_DIFF = [ + "fuzzy_match_percent", + "matched_code_percent", + "matched_data_percent", + "complete_code_percent", + "complete_data_percent", +] + +FUNCTION_KEYS_TO_DIFF = [ + "fuzzy_match_percent", +] + +type Change = Tuple[str, str, float, float] + + +def get_changes(changes_file: str) -> list[Change]: + changes_file = os.path.relpath(changes_file, root_dir) + with open(changes_file, "r") as f: + changes_json = json.load(f) + + regressions = [] + progressions = [] + + def diff_key(object_name: str, object: dict, key: str): + from_value = object.get("from", {}).get(key, 0.0) + to_value = object.get("to", {}).get(key, 0.0) + key = key.removesuffix("_percent") + change = (object_name, key, from_value, to_value) + if from_value > to_value: + regressions.append(change) + elif to_value > from_value: + progressions.append(change) + + for key in UNIT_KEYS_TO_DIFF: + diff_key(None, changes_json, key) + + for unit in changes_json.get("units", []): + unit_name = unit["name"] + for key in UNIT_KEYS_TO_DIFF: + diff_key(unit_name, unit, key) + # Ignore sections + for func in unit.get("functions", []): + func_name = func["name"] + for key in FUNCTION_KEYS_TO_DIFF: + diff_key(func_name, func, key) + + return regressions, progressions + + +def generate_changes_plaintext(changes: list[Change]) -> str: + if len(changes) == 0: + return "" + + table_total_width = 136 + percents_max_len = 7 + 4 + 7 + key_max_len = max(len(key) for _, key, _, _ in changes) + name_max_len = max(len(name or "Total") for name, _, _, _ in changes) + max_width_for_name_col = table_total_width - 3 - key_max_len - 3 - percents_max_len + name_max_len = min(max_width_for_name_col, name_max_len) + + out_lines = [] + for name, key, from_value, to_value in changes: + if name is None: + name = "Total" + if len(name) > name_max_len: + name = name[: name_max_len - len("[...]")] + "[...]" + out_lines.append( + f"{name:>{name_max_len}} | {key:<{key_max_len}} | {from_value:6.2f}% -> {to_value:5.2f}%" + ) + + return "\n".join(out_lines) + + +def generate_changes_markdown(changes: list[Change], description: str) -> str: + if len(changes) == 0: + return "" + + out_lines = [] + name_max_len = 100 + + out_lines.append("
") + out_lines.append( + f"Detected {len(changes)} {description} compared to the base:" + ) + out_lines.append("") # Must include a blank line before a table + out_lines.append("| Name | Type | Before | After |") + out_lines.append("| ---- | ---- | ------ | ----- |") + + for name, key, from_value, to_value in changes: + if name is None: + name = "Total" + else: + if len(name) > name_max_len: + name = name[: name_max_len - len("...")] + "..." + name = f"`{name}`" # Surround with backticks + key = key.replace("_", " ").capitalize() + out_lines.append(f"| {name} | {key} | {from_value:.2f}% | {to_value:.2f}% |") + + out_lines.append("
") + + return "\n".join(out_lines) + + +def main(): + parser = ArgumentParser(description="Format objdiff-cli report changes.") + parser.add_argument( + "report_changes_file", + type=Path, + help="""path to the JSON file containing the changes, generated by objdiff-cli.""", + ) + parser.add_argument( + "-o", + "--output", + type=Path, + help="""Output file (prints to console if unspecified)""", + ) + parser.add_argument( + "--all", + action="store_true", + help="""Includes progressions as well.""", + ) + args = parser.parse_args() + + regressions, progressions = get_changes(args.report_changes_file) + + if args.output: + markdown_output = generate_changes_markdown(regressions, "regressions") + if args.all: + markdown_output += generate_changes_markdown(progressions, "progressions") + with open(args.output, "w", encoding="utf-8") as f: + f.write(markdown_output) + else: + if args.all: + changes = progressions + regressions + else: + changes = regressions + text_output = generate_changes_plaintext(changes) + print(text_output) + + +if __name__ == "__main__": + main() diff --git a/tools/project.py b/tools/project.py index bb282674e..6c5d07af5 100644 --- a/tools/project.py +++ b/tools/project.py @@ -1215,6 +1215,81 @@ def generate_build_ninja( order_only="post-build", ) + n.comment("Phony edge that will always be considered dirty by ninja.") + n.comment( + "This can be used as an implicit to a target that should always be rerun, ignoring file modified times." + ) + n.build( + outputs="always", + rule="phony", + ) + n.newline() + + ### + # Regression test progress reports + ### + report_baseline_path = build_path / "baseline.json" + report_changes_path = build_path / "report_changes.json" + changes_fmt = config.tools_dir / "changes_fmt.py" + regressions_md = build_path / "regressions.md" + n.comment( + "Create a baseline progress report for later match regression testing" + ) + n.build( + outputs=report_baseline_path, + rule="report", + implicit=[objdiff, "all_source", "always"], + order_only="post-build", + ) + n.build( + outputs="baseline", + rule="phony", + inputs=report_baseline_path, + ) + n.comment("Check for any match regressions against the baseline") + n.comment("Will fail if no baseline has been created") + n.rule( + name="report_changes", + command=f"{objdiff} report changes --format json-pretty {report_baseline_path} $in -o $out", + description="CHANGES", + ) + n.build( + outputs=report_changes_path, + rule="report_changes", + inputs=report_path, + implicit=[objdiff, "always"], + ) + n.rule( + name="changes_fmt", + command=f"$python {changes_fmt} $args $in", + description="CHANGESFMT", + ) + n.build( + outputs="changes", + rule="changes_fmt", + inputs=report_changes_path, + implicit=changes_fmt, + ) + n.build( + outputs="changes_all", + rule="changes_fmt", + inputs=report_changes_path, + implicit=changes_fmt, + variables={"args": "--all"}, + ) + n.rule( + name="changes_md", + command=f"$python {changes_fmt} $in -o $out", + description="CHANGESFMT $out", + ) + n.build( + outputs=regressions_md, + rule="changes_md", + inputs=report_changes_path, + implicit=changes_fmt, + ) + n.newline() + ### # Helper tools ###