mirror of https://github.com/astral-sh/ruff
Add a benchmarking script for the formatter CLI (#7340)
## Summary This PR adds a benchmarking script for the formatter, which benchmarks the Ruff formatter against Black, yapf, and autopep8. Three benchmarks are included: 1. Format everything. 2. Format everything, but use a single thread. 3. Format everything, but `--check` (don't write to disk). There's some nuance in figuring out the right combination of arguments to each command, but the _main_ nuance is to ensure that we always run the given formatter (and modify the target repo in-place) prior to benchmarking it, so that the formatters aren't disadvantaged by the existing formatting of the target repo. (E.g.: prior to benchmarking Black's preview style, we need to make sure we format the target repo with Black's preview style; otherwise, preview style appears much slower.) Part of https://github.com/astral-sh/ruff/issues/7309.
This commit is contained in:
Charlie Marsh
GitHub
parent
f0f7ea7502
commit
4df9e07a79
File diff suppressed because it is too large
Load Diff
|
|
@ -6,70 +6,31 @@ authors = ["Charles Marsh <charlie.r.marsh@gmail.com>"]
|
|||
|
||||
[tool.poetry.dependencies]
|
||||
python = ">=3.10,<3.12"
|
||||
autoflake = "^2.0.0"
|
||||
flake8 = "^6.0.0"
|
||||
pycodestyle = "^2.10.0"
|
||||
pyflakes = "^3.0.1"
|
||||
pylint = "^2.15.10"
|
||||
black = "^22.12.0"
|
||||
isort = "^5.11.4"
|
||||
flake8-2020 = { version = "*", optional = true }
|
||||
flake8-annotations = { version = "*", optional = true }
|
||||
flake8-bandit = { version = "*", optional = true }
|
||||
flake8-blind-except = { version = "*", optional = true }
|
||||
# flake8-boolean-trap = { version = "*", optional = true }
|
||||
flake8-bugbear = { version = "*", optional = true }
|
||||
flake8-builtins = { version = "*", optional = true }
|
||||
flake8-commas = { version = "*", optional = true }
|
||||
flake8-comprehensions = { version = "*", optional = true }
|
||||
flake8-datetimez = { version = "*", optional = true }
|
||||
flake8-debugger = { version = "*", optional = true }
|
||||
flake8-docstrings = { version = "*", optional = true }
|
||||
# flake8-eradicate = { version = "*", optional = true }
|
||||
flake8-errmsg = { version = "*", optional = true }
|
||||
flake8-implicit-str-concat = { version = "*", optional = true }
|
||||
# flake8-import-conventions = { version = "*", optional = true }
|
||||
flake8-isort = { version = "*", optional = true }
|
||||
flake8-pie = { version = "*", optional = true }
|
||||
flake8-print = { version = "*", optional = true }
|
||||
flake8-quotes = { version = "*", optional = true }
|
||||
flake8-return = { version = "*", optional = true }
|
||||
flake8-simplify = { version = "*", optional = true }
|
||||
flake8-super = { version = "*", optional = true }
|
||||
flake8-tidy-imports = { version = "*", optional = true }
|
||||
pandas-vet = { version = "*", optional = true }
|
||||
pep8-naming = { version = "*", optional = true }
|
||||
autoflake = "*"
|
||||
autopep8 = "*"
|
||||
black = "*"
|
||||
flake8 = "*"
|
||||
isort = "*"
|
||||
pycodestyle = "*"
|
||||
pyflakes = "*"
|
||||
pylint = "*"
|
||||
yapf = "*"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
|
||||
[tool.poetry.extras]
|
||||
plugins = [
|
||||
"flake8-2020",
|
||||
"flake8-annotations",
|
||||
"flake8-bandit",
|
||||
"flake8-blind-except",
|
||||
# "flake8-boolean-trap",
|
||||
"flake8-bugbear",
|
||||
"flake8-builtins",
|
||||
"flake8-commas",
|
||||
"flake8-comprehensions",
|
||||
"flake8-datetimez",
|
||||
"flake8-debugger",
|
||||
"flake8-docstrings",
|
||||
# "flake8-eradicate",
|
||||
"flake8-errmsg",
|
||||
"flake8-implicit-str-concat",
|
||||
# "flake8-import-conventions",
|
||||
"flake8-isort",
|
||||
"flake8-pie",
|
||||
"flake8-print",
|
||||
"flake8-quotes",
|
||||
"flake8-return",
|
||||
"flake8-simplify",
|
||||
"flake8-super",
|
||||
"flake8-tidy-imports",
|
||||
"pandas-vet",
|
||||
"pep8-naming",
|
||||
formatter = [
|
||||
"black",
|
||||
"yapf",
|
||||
"autopep8",
|
||||
]
|
||||
linter = [
|
||||
"autoflake",
|
||||
"flake8",
|
||||
"pycodestyle",
|
||||
"pyflakes",
|
||||
"pylint",
|
||||
"isort",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env sh
|
||||
|
||||
###
|
||||
# Benchmark the Ruff formatter's performance against a variety of similar tools.
|
||||
#
|
||||
# Expects to be run from the repo root after invoking `cargo build --release`,
|
||||
# in an environment with access to `black`, `autopep8`, and `yapf` (most recently:
|
||||
# `black` v23.9.1, `autopep8` v2.0.4, and `yapf` v0.40.1).
|
||||
#
|
||||
# Example usage:
|
||||
#
|
||||
# ./scripts/benchmarks/run_formatter.sh ~/workspace/zulip
|
||||
###
|
||||
|
||||
TARGET_DIR=${1}
|
||||
|
||||
# In each case, ensure that we format the code in-place before invoking a given tool. This ensures
|
||||
# a fair comparison across tools, since every tool is then running on a repository that already
|
||||
# matches that tool's desired formatting.
|
||||
#
|
||||
# For example, if we're benchmarking Black's preview style, we first run `black --preview` over the
|
||||
# target directory, thus ensuring that we're benchmarking preview style against a codebase that
|
||||
# already conforms to it. The same goes for yapf, autoepp8, etc.
|
||||
|
||||
# Benchmark 1: Write to disk.
|
||||
hyperfine --ignore-failure \
|
||||
--prepare "./target/release/ruff format ${TARGET_DIR}" \
|
||||
"./target/release/ruff format ${TARGET_DIR}" \
|
||||
--prepare "BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --safe" \
|
||||
"BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --safe" \
|
||||
--prepare "BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --fast" \
|
||||
"BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --fast" \
|
||||
--prepare "BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --safe --preview" \
|
||||
"BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --safe --preview" \
|
||||
--prepare "BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --fast --preview" \
|
||||
"BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --fast --preview" \
|
||||
--prepare "autopep8 ${TARGET_DIR} --recursive --in-place" \
|
||||
"autopep8 ${TARGET_DIR} --recursive --in-place" \
|
||||
--prepare "yapf ${TARGET_DIR} --parallel --recursive --in-place" \
|
||||
"yapf ${TARGET_DIR} --parallel --recursive --in-place"
|
||||
|
||||
# Benchmark 2: Write to disk, but only use one thread.
|
||||
hyperfine --ignore-failure \
|
||||
--prepare "./target/release/ruff format ${TARGET_DIR}" \
|
||||
"RAYON_NUM_THREADS=1 ./target/release/ruff format ${TARGET_DIR}" \
|
||||
--prepare "BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --safe" \
|
||||
"BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --workers=1 --safe" \
|
||||
--prepare "BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --fast" \
|
||||
"BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --workers=1 --fast" \
|
||||
--prepare "BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --safe --preview" \
|
||||
"BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --workers=1 --safe --preview" \
|
||||
--prepare "BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --fast --preview" \
|
||||
"BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --workers=1 --fast --preview" \
|
||||
--prepare "autopep8 ${TARGET_DIR} --recursive --in-place" \
|
||||
"autopep8 ${TARGET_DIR} --in-place --recursive --jobs=1" \
|
||||
--prepare "yapf ${TARGET_DIR} --parallel --recursive --in-place" \
|
||||
"yapf ${TARGET_DIR} --recursive --in-place"
|
||||
|
||||
# Benchmark 3: Check formatting, but don't write to disk.
|
||||
hyperfine --ignore-failure \
|
||||
--prepare "./target/release/ruff format ${TARGET_DIR}" \
|
||||
"./target/release/ruff format ${TARGET_DIR} --check" \
|
||||
--prepare "BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --safe" \
|
||||
"BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --check --safe" \
|
||||
--prepare "BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --fast" \
|
||||
"BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --check --fast" \
|
||||
--prepare "BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --safe --preview" \
|
||||
"BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --check --safe --preview" \
|
||||
--prepare "BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --fast --preview" \
|
||||
"BLACK_CACHE_DIR=/dev/null black ${TARGET_DIR} --check --fast --preview" \
|
||||
--prepare "autopep8 ${TARGET_DIR} --recursive --in-place" \
|
||||
"autopep8 ${TARGET_DIR} --recursive --diff" \
|
||||
--prepare "yapf ${TARGET_DIR} --parallel --recursive --in-place" \
|
||||
"yapf ${TARGET_DIR} --parallel --recursive --quiet"
|
||||
Loading…
Reference in New Issue