.

scripts/ecosystem-testing/compare.py

@@ -1,5 +1,7 @@
 import argparse
 import difflib
+import json
+import re
 import sys
 from pathlib import Path
 
@@ -8,34 +10,114 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("base", type=Path)
     parser.add_argument("branch", type=Path)
+    parser.add_argument(
+        "--markdown",
+        action="store_true",
+        help="Output in markdown format with collapsible sections",
+    )
+    parser.add_argument(
+        "--show-failed",
+        action="store_true",
+        help="Show failed packages",
+    )
     args = parser.parse_args()
 
+    redact_time = re.compile(r"(0\.)?(\d+)ms")
+
     total = 0
-    differences = 0
-    for package in args.base.iterdir():
-        if not package.is_dir():
+    successful = 0
+    differences = []
+    for package_dir in args.base.iterdir():
+        if not package_dir.is_dir():
             continue
-        package_branch = args.branch.joinpath(package.name)
+        package = package_dir.name
+        package_branch = args.branch.joinpath(package)
         if not package_branch.is_dir():
             print(f"Package {package} not found in branch")
             continue
-        stdout = package.joinpath("stdout.txt").read_text()
-        stdout_branch = package_branch.joinpath("stdout.txt").read_text()
-        if stdout != stdout_branch:
-            differences += 1
-            print("--------------------------------")
-            print(f"Package {package}")
-            sys.stdout.writelines(
-                difflib.unified_diff(
-                    stdout.splitlines(keepends=True),
-                    stdout_branch.splitlines(keepends=True),
-                    fromfile="base",
-                    tofile="branch",
-                )
-            )
+
         total += 1
 
-    print(f"Different packages: {differences}/{total}")
+        stdout = package_dir.joinpath("stdout.txt").read_text()
+        stderr = package_dir.joinpath("stderr.txt").read_text()
+        stderr = redact_time.sub(r"[TIME]", stderr)
+        summary = package_dir.joinpath("summary.json").read_text()
+        stdout_branch = package_branch.joinpath("stdout.txt").read_text()
+        stderr_branch = package_branch.joinpath("stderr.txt").read_text()
+        stderr_branch = redact_time.sub(r"[TIME]", stderr_branch)
+
+        if json.loads(summary)["exit_code"] == 0:
+            successful += 1
+        elif not args.show_failed:
+            # Don't show differences in the error messages by default
+            continue
+
+        if stdout != stdout_branch or stderr != stderr_branch:
+            differences.append((package, stdout, stdout_branch, stderr, stderr_branch))
+
+    if args.markdown:
+        print("# Ecosystem testing report")
+        print(
+            "Dataset: `uv pip compile` on each of the top 15k PyPI packages on Python 3.13 with `--no-build`, "
+            "a handful of pathological cases filtered. "
+            "Only success resolutions can be compared.\n"
+        )
+        print(f"Successfully resolved packages: {successful}/{total}\n")
+        print(f"Different packages: {len(differences)}/{total}\n")
+
+        for package, stdout, stdout_branch, stderr, stderr_branch in differences:
+            print(f"\n<details>\n<summary>{package}</summary>\n")
+            if stdout != stdout_branch:
+                print("```diff")
+                sys.stdout.writelines(
+                    difflib.unified_diff(
+                        stdout.splitlines(keepends=True),
+                        stdout_branch.splitlines(keepends=True),
+                        fromfile="base",
+                        tofile="branch",
+                        # Show the dependencies in full
+                        n=999999,
+                    )
+                )
+                print("```")
+            if stderr != stderr_branch:
+                print("```diff")
+                sys.stdout.writelines(
+                    difflib.unified_diff(
+                        stderr.splitlines(keepends=True),
+                        stderr_branch.splitlines(keepends=True),
+                        fromfile="base",
+                        tofile="branch",
+                        # Show the log in full
+                        n=999999,
+                    )
+                )
+                print("```")
+            print("</details>")
+    else:
+        for package, stdout, stdout_branch, stderr, stderr_branch in differences:
+            print("--------------------------------")
+            print(f"Package {package}")
+            if stdout != stdout_branch:
+                sys.stdout.writelines(
+                    difflib.unified_diff(
+                        stdout.splitlines(keepends=True),
+                        stdout_branch.splitlines(keepends=True),
+                        fromfile="base",
+                        tofile="branch",
+                    )
+                )
+            if stderr != stderr_branch:
+                sys.stdout.writelines(
+                    difflib.unified_diff(
+                        stderr.splitlines(keepends=True),
+                        stderr_branch.splitlines(keepends=True),
+                        fromfile="base",
+                        tofile="branch",
+                    )
+                )
+        print(f"Successfully resolved packages: {successful}/{total}")
+        print(f"Different packages: {len(differences)}/{total}")
 
 
 if __name__ == "__main__":