From f852d986f3d787e76d01110c11f3f8adab46735a Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Thu, 18 Jan 2024 13:18:37 -0500 Subject: [PATCH] Add an incremental resolution benchmark (#954) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This adds a benchmark in which we reuse the lockfile, but add a new dependency to the input requirements. Running `python -m scripts.bench --poetry --puffin --pip-compile scripts/requirements/trio.in --benchmark resolve-warm --benchmark resolve-incremental`: ```text Benchmark 1: pip-compile (resolve-warm) Time (mean ± σ): 1.169 s ± 0.023 s [User: 0.675 s, System: 0.112 s] Range (min … max): 1.129 s … 1.198 s 10 runs Benchmark 2: poetry (resolve-warm) Time (mean ± σ): 610.7 ms ± 10.4 ms [User: 528.1 ms, System: 60.3 ms] Range (min … max): 599.9 ms … 632.6 ms 10 runs Benchmark 3: puffin (resolve-warm) Time (mean ± σ): 19.3 ms ± 0.6 ms [User: 13.5 ms, System: 13.1 ms] Range (min … max): 17.9 ms … 22.1 ms 122 runs Summary 'puffin (resolve-warm)' ran 31.63 ± 1.19 times faster than 'poetry (resolve-warm)' 60.53 ± 2.37 times faster than 'pip-compile (resolve-warm)' Benchmark 1: pip-compile (resolve-incremental) Time (mean ± σ): 1.554 s ± 0.059 s [User: 0.974 s, System: 0.130 s] Range (min … max): 1.473 s … 1.652 s 10 runs Benchmark 2: poetry (resolve-incremental) Time (mean ± σ): 474.2 ms ± 2.4 ms [User: 411.7 ms, System: 54.0 ms] Range (min … max): 470.6 ms … 477.7 ms 10 runs Benchmark 3: puffin (resolve-incremental) Time (mean ± σ): 28.0 ms ± 1.1 ms [User: 21.7 ms, System: 14.6 ms] Range (min … max): 26.7 ms … 34.4 ms 89 runs Summary 'puffin (resolve-incremental)' ran 16.94 ± 0.67 times faster than 'poetry (resolve-incremental)' 55.52 ± 3.02 times faster than 'pip-compile (resolve-incremental)' ``` --- scripts/bench/__main__.py | 169 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/scripts/bench/__main__.py b/scripts/bench/__main__.py index b7158c2cb..2f7e2e2e0 100644 --- a/scripts/bench/__main__.py +++ b/scripts/bench/__main__.py @@ -31,6 +31,7 @@ import enum import logging import os.path import shlex +import shutil import subprocess import tempfile import typing @@ -45,6 +46,7 @@ class Benchmark(enum.Enum): RESOLVE_COLD = "resolve-cold" RESOLVE_WARM = "resolve-warm" + RESOLVE_INCREMENTAL = "resolve-incremental" INSTALL_COLD = "install-cold" INSTALL_WARM = "install-warm" @@ -102,6 +104,12 @@ class Hyperfine(typing.NamedTuple): subprocess.check_call(args) +# The requirement to use when benchmarking an incremental resolution. +# Ideally, this requirement is compatible with all requirements files, but does not +# appear in any resolutions. +INCREMENTAL_REQUIREMENT = "django" + + class Suite(abc.ABC): """Abstract base class for packaging tools.""" @@ -118,6 +126,8 @@ class Suite(abc.ABC): return self.resolve_cold(requirements_file, cwd=cwd) case Benchmark.RESOLVE_WARM: return self.resolve_warm(requirements_file, cwd=cwd) + case Benchmark.RESOLVE_INCREMENTAL: + return self.resolve_incremental(requirements_file, cwd=cwd) case Benchmark.INSTALL_COLD: return self.install_cold(requirements_file, cwd=cwd) case Benchmark.INSTALL_WARM: @@ -141,6 +151,15 @@ class Suite(abc.ABC): however, the cache directory is _not_ cleared between runs. """ + @abc.abstractmethod + def resolve_incremental(self, requirements_file: str, *, cwd: str) -> Command | None: + """Resolve a modified lockfile using pip-tools, from a warm cache. + + The resolution is performed with an existing lock file, and the cache directory + is _not_ cleared between runs. However, a new dependency is added to the set + of input requirements, which does not appear in the lock file. + """ + @abc.abstractmethod def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None: """Install a set of dependencies using pip-tools, from a cold cache. @@ -198,6 +217,49 @@ class PipCompile(Suite): ], ) + def resolve_incremental(self, requirements_file: str, *, cwd: str) -> Command | None: + cache_dir = os.path.join(cwd, ".cache") + baseline = os.path.join(cwd, "baseline.txt") + + # First, perform a cold resolution, to ensure that the lock file exists. + # TODO(charlie): Make this a `setup`. + subprocess.check_call( + [ + self.path, + os.path.abspath(requirements_file), + "--cache-dir", + cache_dir, + "--output-file", + baseline, + ], + cwd=cwd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + assert os.path.exists(baseline), f"Lock file doesn't exist at: {baseline}" + + input_file = os.path.join(cwd, "requirements.in") + output_file = os.path.join(cwd, "requirements.txt") + + # Add a dependency to the requirements file. + with open(input_file, "w") as fp1: + fp1.write(f"{INCREMENTAL_REQUIREMENT}\n") + with open(requirements_file) as fp2: + fp1.writelines(fp2.readlines()) + + return Command( + name=f"{self.name} ({Benchmark.RESOLVE_INCREMENTAL.value})", + prepare=f"rm -f {output_file} && cp {baseline} {output_file}", + command=[ + self.path, + input_file, + "--cache-dir", + cache_dir, + "--output-file", + output_file, + ], + ) + def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None: ... @@ -206,6 +268,7 @@ class PipCompile(Suite): class PipSync(Suite): + def __init__(self, path: str | None = None) -> None: self.name = path or "pip-sync" self.path = path or "pip-sync" @@ -216,6 +279,9 @@ class PipSync(Suite): def resolve_warm(self, requirements_file: str, *, cwd: str) -> Command | None: ... + def resolve_incremental(self, requirements_file: str, *, cwd: str) -> Command | None: + ... + def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None: cache_dir = os.path.join(cwd, ".cache") venv_dir = os.path.join(cwd, ".venv") @@ -346,6 +412,62 @@ class Poetry(Suite): ], ) + def resolve_incremental(self, requirements_file: str, *, cwd: str) -> Command | None: + self.setup(requirements_file, cwd=cwd) + + poetry_lock = os.path.join(cwd, "poetry.lock") + assert not os.path.exists( + poetry_lock + ), f"Lock file already exists at: {poetry_lock}" + + # Run a resolution, to ensure that the lock file exists. + # TODO(charlie): Make this a `setup`. + subprocess.check_call( + [self.path, "lock"], + cwd=cwd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + assert os.path.exists(poetry_lock), f"Lock file doesn't exist at: {poetry_lock}" + + # Add a dependency to the requirements file. + with open(os.path.join(cwd, "pyproject.toml"), "rb") as fp: + pyproject = tomli.load(fp) + + # Add the dependencies to the pyproject.toml. + pyproject["tool"]["poetry"]["dependencies"].update( + { + INCREMENTAL_REQUIREMENT: "*", + } + ) + + with open(os.path.join(cwd, "pyproject.toml"), "wb") as fp: + tomli_w.dump(pyproject, fp) + + # Store the baseline lock file. + baseline = os.path.join(cwd, "baseline.lock") + shutil.copyfile(poetry_lock, baseline) + + poetry_lock = os.path.join(cwd, "poetry.lock") + config_dir = os.path.join(cwd, "config", "pypoetry") + cache_dir = os.path.join(cwd, "cache", "pypoetry") + data_dir = os.path.join(cwd, "data", "pypoetry") + + return Command( + name=f"{self.name} ({Benchmark.RESOLVE_INCREMENTAL.value})", + prepare=f"rm -f {poetry_lock} && cp {baseline} {poetry_lock}", + command=[ + f"POETRY_CONFIG_DIR={config_dir}", + f"POETRY_CACHE_DIR={cache_dir}", + f"POETRY_DATA_DIR={data_dir}", + self.path, + "lock", + "--no-update", + "--directory", + cwd, + ], + ) + def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None: self.setup(requirements_file, cwd=cwd) @@ -482,6 +604,53 @@ class Puffin(Suite): ], ) + def resolve_incremental(self, requirements_file: str, *, cwd: str) -> Command | None: + cache_dir = os.path.join(cwd, ".cache") + baseline = os.path.join(cwd, "baseline.txt") + + # First, perform a cold resolution, to ensure that the lock file exists. + # TODO(charlie): Make this a `setup`. + subprocess.check_call( + [ + self.path, + "pip", + "compile", + os.path.abspath(requirements_file), + "--cache-dir", + cache_dir, + "--output-file", + baseline, + ], + cwd=cwd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + assert os.path.exists(baseline), f"Lock file doesn't exist at: {baseline}" + + input_file = os.path.join(cwd, "requirements.in") + output_file = os.path.join(cwd, "requirements.txt") + + # Add a dependency to the requirements file. + with open(input_file, "w") as fp1: + fp1.write(f"{INCREMENTAL_REQUIREMENT}\n") + with open(requirements_file) as fp2: + fp1.writelines(fp2.readlines()) + + return Command( + name=f"{self.name} ({Benchmark.RESOLVE_INCREMENTAL.value})", + prepare=f"rm -f {output_file} && cp {baseline} {output_file}", + command=[ + self.path, + "pip", + "compile", + input_file, + "--cache-dir", + cache_dir, + "--output-file", + output_file, + ], + ) + def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None: cache_dir = os.path.join(cwd, ".cache") venv_dir = os.path.join(cwd, ".venv")