#!/usr/bin/env -S uv run --script # /// script # requires-python = ">=3.13" # /// import argparse import difflib import json import re import sys from pathlib import Path def main(): parser = argparse.ArgumentParser() parser.add_argument("base", type=Path) parser.add_argument("branch", type=Path) parser.add_argument("--mode", choices=["compile", "lock", "pyproject-toml"]) parser.add_argument( "--markdown", action="store_true", ) parser.add_argument( "--limit", type=int, default=None, ) args = parser.parse_args() # Supress noise from fluctuations in execution time redact_time = re.compile(r"(0\.)?(\d+)ms|(\d+).(\d+)s") parameters = json.loads(args.base.joinpath("parameters.json").read_text()) total = 0 successful = 0 differences = [] files = sorted(dir for dir in args.base.iterdir() if dir.is_dir()) for package_dir in files[: args.limit]: package = package_dir.name package_branch = args.branch.joinpath(package) if not package_branch.is_dir(): print(f"Package {package} not found in branch") continue total += 1 summary = package_dir.joinpath("summary.json").read_text() if json.loads(summary)["exit_code"] == 0: successful += 1 else: # Don't show differences in the error messages, # also `uv.lock` doesn't exist for failed resolutions continue if args.mode == "compile": resolution = package_dir.joinpath("stdout.txt").read_text() else: resolution = package_dir.joinpath("uv.lock").read_text() if package_dir.joinpath("stdout.txt").read_text().strip(): raise RuntimeError(f"Stdout not empty (base): {package}") stderr = package_dir.joinpath("stderr.txt").read_text() stderr = redact_time.sub(r"[TIME]", stderr) if args.mode == "compile": resolution_branch = package_branch.joinpath("stdout.txt").read_text() else: resolution_branch = package_branch.joinpath("uv.lock").read_text() if package_branch.joinpath("stdout.txt").read_text().strip(): raise RuntimeError(f"Stdout not empty (branch): {package}") stderr_branch = package_branch.joinpath("stderr.txt").read_text() stderr_branch = redact_time.sub(r"[TIME]", stderr_branch) if resolution != resolution_branch or stderr != stderr_branch: differences.append( (package, resolution, resolution_branch, stderr, stderr_branch) ) if args.markdown: print( "## Ecosystem testing report " f"({args.mode.replace('pyproject-toml', 'pyproject.toml')})" ) if args.mode == "pyproject-toml": print( " * Dataset: A set of top level `pyproject.toml` from GitHub projects popular in 2025. " + "Only `pyproject.toml` files with a `[project]` section and static dependencies are included." ) else: print( " * Dataset: The top 15k PyPI packages. A handful of pathological cases were filtered out." ) print( " * Command: " + f"`{'uv pip compile' if args.mode == 'compile' else 'uv lock'}` with `--no-build` " + f"on Python {parameters['python']} " + ( "pinned to the latest package version. " if parameters["latest"] else ". " ) ) print( f" * Successfully resolved packages: {successful}/{total} ({successful / total:.0%}). " + "Only success resolutions can be compared." ) print(f" * Different packages: {len(differences)}/{successful}") for ( package, resolution, resolution_branch, stderr, stderr_branch, ) in differences: if args.mode == "compile": context_window = 999999 else: context_window = 3 print(f"\n
\n{package}\n") if resolution != resolution_branch: print("```diff") sys.stdout.writelines( difflib.unified_diff( resolution.splitlines(keepends=True), resolution_branch.splitlines(keepends=True), fromfile="base", tofile="branch", # Show the dependencies in full n=context_window, ) ) print("```") if stderr != stderr_branch: print("```diff") sys.stdout.writelines( difflib.unified_diff( stderr.splitlines(keepends=True), stderr_branch.splitlines(keepends=True), fromfile="base", tofile="branch", # Show the log in full n=context_window, ) ) print("```") print("
\n") else: for ( package, resolution, resolution_branch, stderr, stderr_branch, ) in differences: print("--------------------------------") print(f"Package {package}") if resolution != resolution_branch: sys.stdout.writelines( difflib.unified_diff( resolution.splitlines(keepends=True), resolution_branch.splitlines(keepends=True), fromfile="base", tofile="branch", ) ) if stderr != stderr_branch: sys.stdout.writelines( difflib.unified_diff( stderr.splitlines(keepends=True), stderr_branch.splitlines(keepends=True), fromfile="base", tofile="branch", ) ) print( f"Successfully resolved packages: {successful}/{total} ({successful / total:.0%})" ) print(f"Different packages: {len(differences)}/{successful}") if __name__ == "__main__": main()