diff --git a/BENCHMARKS.md b/BENCHMARKS.md new file mode 100644 index 000000000..47ddf3d37 --- /dev/null +++ b/BENCHMARKS.md @@ -0,0 +1,102 @@ +# Benchmarks + +All benchmarks were computed on macOS, and come with a few important caveats: + +- Benchmark performance may vary dramatically across different operating systems and filesystems. + In particular, Puffin uses different installation strategies based on the underlying filesystem's + capabilities. (For example, Puffin uses reflinking on macOS, and hardlinking on Linux.) +- Benchmark performance may vary dramatically depending on the set of packages being installed. + For example, a resolution that requires building a single intensive source distribution may appear + very similar across tools, since the bottleneck is tool-agnostic. +- Unlike Poetry, both Puffin and pip-tools do _not_ generate multi-platform lockfiles. As such, + Poetry is (by design) doing significantly more work than other tools in the resolution benchmarks. + Poetry is included for completeness, as many projects may not _need_ a multi-platform lockfile. + However, it's critical to understand that benchmarking Puffin's resolution time against Poetry is + an unfair comparison. (Benchmarking installation, however, _is_ a fair comparison.) + +This document benchmarks against Trio's `docs-requirements.in`, as a representative example of a +real-world project. + +In each case, a smaller bar (i.e., lower) is better. + +## Warm Installation + +Benchmarking package installation (e.g., `puffin pip sync`) with a warm cache. This is equivalent +to removing and recreating a virtual environment, and then populating it with dependencies that +you've installed previously on the same machine. + +![](https://github.com/astral-sh/ruff/assets/1309177/6ceea7aa-4813-4ea8-8c95-b8013d702cf4) + +## Cold Installation + +Benchmarking package installation (e.g., `puffin pip sync`) with a cold cache. This is equivalent +to running `puffin pip sync` on a new machine or in CI (assuming that the package manager cache is +not shared across runs). + +![](https://github.com/astral-sh/ruff/assets/1309177/c960d6fd-ec34-467e-9aa2-d4e6713abed0) + +## Warm Resolution + +Benchmarking dependency resolution (e.g., `puffin pip compile`) with a warm cache, but no existing +lockfile. This is equivalent to blowing away an existing `requirements.txt` file to regenerate it +from a `requirements.in` file. + +![](https://github.com/astral-sh/ruff/assets/1309177/aab99181-e54e-4bdb-9ce6-15b018ef8466) + +## Cold Resolution + +Benchmarking dependency resolution (e.g., `puffin pip compile`) with a cold cache. This is +equivalent to running `puffin pip compile` on a new machine or in CI (assuming that the package +manager cache is not shared across runs). + +![](https://github.com/astral-sh/ruff/assets/1309177/a6075ebc-bb8f-46db-a3b4-14ee5f713565) + +## Reproduction + +All benchmarks were generated using the `scripts/bench/__main__.py` script, which wraps +[`hyperfine`](https://github.com/sharkdp/hyperfine) to facilitate benchmarking Puffin +against a variety of other tools. + +The benchmark script itself has a several requirements: + +- A local Puffin release build (`cargo build --release`). +- A virtual environment with the script's own dependencies installed (`puffin venv && puffin pip sync scripts/bench/requirements.txt`). +- The [`hyperfine`](https://github.com/sharkdp/hyperfine) command-line tool installed on your system. + +To benchmark Puffin's resolution against pip-compile and Poetry: + +```shell +python -m scripts.bench \ + --puffin \ + --poetry \ + --pip-compile \ + --benchmark resolve-warm \ + scripts/requirements/trio.in \ + --json +``` + +To benchmark Puffin's installation against pip-sync and Poetry: + +```shell +python -m scripts.bench \ + --puffin \ + --poetry \ + --pip-sync \ + --benchmark resolve-warm \ + scripts/requirements/compiled/trio.txt \ + --json +``` + +After running the benchmark script, you can generate the corresponding graph via: + +```shell +cargo run -p puffin-dev render-benchmarks resolve-warm.json --title "Warm Resolution" +cargo run -p puffin-dev render-benchmarks resolve-cold.json --title "Cold Resolution" +cargo run -p puffin-dev render-benchmarks install-warm.json --title "Warm Installation" +cargo run -p puffin-dev render-benchmarks install-cold.json --title "Cold Installation" +``` + +## Acknowledgements + +The inclusion of this `BENCHMARKS.md` file was inspired by the excellent benchmarking documentation +in [Orogene](https://github.com/orogene/orogene/blob/472e481b4fc6e97c2b57e69240bf8fe995dfab83/BENCHMARKS.md). diff --git a/Cargo.lock b/Cargo.lock index 85f14c45c..7f673c9d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,6 +133,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + [[package]] name = "arrayvec" version = "0.7.4" @@ -205,7 +211,7 @@ dependencies = [ "futures", "http-content-range", "itertools 0.12.1", - "memmap2", + "memmap2 0.9.4", "reqwest", "thiserror", "tokio", @@ -378,6 +384,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bytemuck" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2490600f404f2b94c167e31d3ed1d5f3c225a0f3b80230053b3e0b7b962bd9" + [[package]] name = "byteorder" version = "1.5.0" @@ -562,6 +574,12 @@ dependencies = [ "cc", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.0" @@ -746,6 +764,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" +[[package]] +name = "data-url" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d7439c3735f405729d52c3fbbe4de140eaf938a1fe47d227c27f8254d4302a5" + [[package]] name = "deranged" version = "0.3.11" @@ -899,6 +923,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "fdeflate" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f9bfee30e4dedf0ab8b422f03af778d9612b63f502710fc500a334ebe2de645" +dependencies = [ + "simd-adler32", +] + [[package]] name = "filetime" version = "0.2.23" @@ -943,6 +976,27 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "fontconfig-parser" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a595cb550439a117696039dfc69830492058211b771a2a165379f2a1a53d84d" +dependencies = [ + "roxmltree 0.19.0", +] + +[[package]] +name = "fontdb" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff20bef7942a72af07104346154a70a70b089c572e454b41bef6eb6cb10e9c06" +dependencies = [ + "fontconfig-parser", + "log", + "memmap2 0.5.10", + "ttf-parser", +] + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1103,6 +1157,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gif" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80792593675e051cf94a4b111980da2ba60d4a83e43e0048c5693baab3977045" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.28.1" @@ -1424,6 +1488,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "imagesize" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b72ad49b554c1728b1e83254a1b1565aea4161e28dabbfa171fc15fe62299caf" + [[package]] name = "indexmap" version = "1.9.3" @@ -1599,6 +1669,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" + [[package]] name = "js-sys" version = "0.3.67" @@ -1618,6 +1694,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "kurbo" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a53776d271cfb873b17c618af0298445c88afc52837f3e948fa3fafd131f449" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "kurbo" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd85a5776cd9500c2e2059c8c76c3b01528566b7fcbaf8098b55a33fc298849b" +dependencies = [ + "arrayvec", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -1764,6 +1858,15 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + [[package]] name = "memmap2" version = "0.9.4" @@ -1844,6 +1947,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", + "simd-adler32", ] [[package]] @@ -2133,6 +2237,12 @@ dependencies = [ "indexmap 2.2.1", ] +[[package]] +name = "pico-args" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" + [[package]] name = "pin-project" version = "1.1.4" @@ -2226,6 +2336,28 @@ dependencies = [ "time", ] +[[package]] +name = "png" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f6c3c3e617595665b8ea2ff95a86066be38fb121ff920a9c0eb282abcd1da5a" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "poloto" +version = "19.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "164dbd541c9832e92fa34452e9c2e98b515a548a3f8549fb2402fe1cd5e46b96" +dependencies = [ + "tagu", +] + [[package]] name = "portable-atomic" version = "1.6.0" @@ -2510,6 +2642,7 @@ dependencies = [ "petgraph", "platform-host", "platform-tags", + "poloto", "puffin-build", "puffin-cache", "puffin-client", @@ -2521,7 +2654,11 @@ dependencies = [ "puffin-resolver", "puffin-traits", "pypi-types", + "resvg", "rustc-hash", + "serde", + "serde_json", + "tagu", "tempfile", "tikv-jemallocator", "tokio", @@ -3000,6 +3137,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rctree" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b42e27ef78c35d3998403c1d26f3efd9e135d3e5121b0a4845cc5cc27547f4f" + [[package]] name = "redox_syscall" version = "0.2.16" @@ -3210,6 +3353,25 @@ dependencies = [ "wasm-timer", ] +[[package]] +name = "resvg" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76888219c0881e22b0ceab06fddcfe83163cd81642bd60c7842387f9c968a72e" +dependencies = [ + "gif", + "jpeg-decoder", + "log", + "pico-args", + "png", + "rgb", + "svgfilters", + "svgtypes 0.10.0", + "tiny-skia", + "usvg", + "usvg-text-layout", +] + [[package]] name = "retry-policies" version = "0.2.1" @@ -3221,6 +3383,15 @@ dependencies = [ "rand", ] +[[package]] +name = "rgb" +version = "0.8.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05aaa8004b64fd573fc9d002f4e632d51ad4f026c2b5ba95fcb6c2f32c2c47d8" +dependencies = [ + "bytemuck", +] + [[package]] name = "ring" version = "0.17.7" @@ -3286,6 +3457,34 @@ dependencies = [ "serde", ] +[[package]] +name = "rosvgtree" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdc23d1ace03d6b8153c7d16f0708cd80b61ee8e80304954803354e67e40d150" +dependencies = [ + "log", + "roxmltree 0.18.1", + "simplecss", + "siphasher", + "svgtypes 0.9.0", +] + +[[package]] +name = "roxmltree" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862340e351ce1b271a378ec53f304a5558f7db87f3769dc655a8f6ecbb68b302" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "roxmltree" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd14fd5e3b777a7422cca79358c57a8f6e3a703d9ac187448d0daf220c2407f" + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -3342,6 +3541,22 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustybuzz" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162bdf42e261bee271b3957691018634488084ef577dddeb6420a9684cab2a6a" +dependencies = [ + "bitflags 1.3.2", + "bytemuck", + "smallvec", + "ttf-parser", + "unicode-bidi-mirroring", + "unicode-ccc", + "unicode-general-category", + "unicode-script", +] + [[package]] name = "ryu" version = "1.0.16" @@ -3503,6 +3718,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "simdutf8" version = "0.1.4" @@ -3515,6 +3736,21 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" +[[package]] +name = "simplecss" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a11be7c62927d9427e9f40f3444d5499d868648e2edbc4e2116de69e7ec0e89d" +dependencies = [ + "log", +] + +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "slab" version = "0.4.9" @@ -3552,6 +3788,15 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "strict-num" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6637bab7722d379c8b41ba849228d680cc12d0a45ba1fa2b48f2a30577a06731" +dependencies = [ + "float-cmp", +] + [[package]] name = "strsim" version = "0.10.0" @@ -3598,6 +3843,36 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2198f991cd549041203080de947415bae45220eab7253c220b87e3188d19f21a" +[[package]] +name = "svgfilters" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "639abcebc15fdc2df179f37d6f5463d660c1c79cd552c12343a4600827a04bce" +dependencies = [ + "float-cmp", + "rgb", +] + +[[package]] +name = "svgtypes" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9ee29c1407a5b18ccfe5f6ac82ac11bab3b14407e09c209a6c1a32098b19734" +dependencies = [ + "kurbo 0.8.3", + "siphasher", +] + +[[package]] +name = "svgtypes" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98ffacedcdcf1da6579c907279b4f3c5492fbce99fbbf227f5ed270a589c2765" +dependencies = [ + "kurbo 0.9.5", + "siphasher", +] + [[package]] name = "syn" version = "1.0.109" @@ -3641,6 +3916,12 @@ dependencies = [ "libc", ] +[[package]] +name = "tagu" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddb6b06d20fba9ed21fca3d696ee1b6e870bca0bcf9fa2971f6ae2436de576a" + [[package]] name = "tap" version = "1.0.1" @@ -3834,6 +4115,31 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-skia" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8493a203431061e901613751931f047d1971337153f96d0e5e363d6dbf6a67" +dependencies = [ + "arrayref", + "arrayvec", + "bytemuck", + "cfg-if", + "png", + "tiny-skia-path", +] + +[[package]] +name = "tiny-skia-path" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adbfb5d3f3dd57a0e11d12f4f13d4ebbbc1b5c15b7ab0a156d030b21da5f677c" +dependencies = [ + "arrayref", + "bytemuck", + "strict-num", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -4096,6 +4402,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ttf-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0609f771ad9c6155384897e1df4d948e692667cc0588548b68eb44d052b27633" + [[package]] name = "typenum" version = "1.17.0" @@ -4117,6 +4429,24 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +[[package]] +name = "unicode-bidi-mirroring" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d12260fb92d52f9008be7e4bca09f584780eb2266dc8fecc6a192bec561694" + +[[package]] +name = "unicode-ccc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2520efa644f8268dce4dcd3050eaa7fc044fca03961e9998ac7e2e92b77cf1" + +[[package]] +name = "unicode-general-category" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2281c8c1d221438e373249e065ca4989c4c36952c211ff21a0ee91c44a3869e7" + [[package]] name = "unicode-ident" version = "1.0.12" @@ -4138,6 +4468,18 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-script" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d817255e1bed6dfd4ca47258685d14d2bdcfbc64fdc9e3819bd5848057b8ecc" + +[[package]] +name = "unicode-vo" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d386ff53b415b7fe27b50bb44679e2cc4660272694b7b6f3326d8480823a94" + [[package]] name = "unicode-width" version = "0.1.11" @@ -4174,6 +4516,39 @@ dependencies = [ "serde", ] +[[package]] +name = "usvg" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b6bb4e62619d9f68aa2d8a823fea2bff302340a1f2d45c264d5b0be170832e" +dependencies = [ + "base64 0.21.7", + "data-url", + "flate2", + "imagesize", + "kurbo 0.9.5", + "log", + "rctree", + "rosvgtree", + "strict-num", +] + +[[package]] +name = "usvg-text-layout" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195386e01bc35f860db024de275a76e7a31afdf975d18beb6d0e44764118b4db" +dependencies = [ + "fontdb", + "kurbo 0.9.5", + "log", + "rustybuzz", + "unicode-bidi", + "unicode-script", + "unicode-vo", + "usvg", +] + [[package]] name = "utf8-width" version = "0.1.7" @@ -4387,6 +4762,12 @@ version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" +[[package]] +name = "weezl" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" + [[package]] name = "which" version = "6.0.0" @@ -4621,6 +5002,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/crates/puffin-dev/Cargo.toml b/crates/puffin-dev/Cargo.toml index d7465be9e..b351321a7 100644 --- a/crates/puffin-dev/Cargo.toml +++ b/crates/puffin-dev/Cargo.toml @@ -36,6 +36,8 @@ puffin-resolver = { path = "../puffin-resolver" } pypi-types = { path = "../pypi-types" } puffin-traits = { path = "../puffin-traits" } +# Any dependencies that are exclusively used in `puffin-dev` should be listed as non-workspace +# dependencies, to ensure that we're forced to think twice before including them in other crates. anstream = { workspace = true } anyhow = { workspace = true } chrono = { workspace = true } @@ -46,7 +48,12 @@ indicatif = { workspace = true } itertools = { workspace = true } owo-colors = { workspace = true } petgraph = { workspace = true } +poloto = { version = "19.1.2" } +resvg = { version = "0.29.0" } rustc-hash = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tagu = { version = "0.1.6" } tempfile = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } diff --git a/crates/puffin-dev/src/main.rs b/crates/puffin-dev/src/main.rs index e6ba87f2e..af47260d6 100644 --- a/crates/puffin-dev/src/main.rs +++ b/crates/puffin-dev/src/main.rs @@ -22,6 +22,7 @@ use resolve_many::ResolveManyArgs; use crate::build::{build, BuildArgs}; use crate::install_many::InstallManyArgs; +use crate::render_benchmarks::RenderBenchmarksArgs; use crate::resolve_cli::ResolveCliArgs; use crate::wheel_metadata::WheelMetadataArgs; @@ -43,6 +44,7 @@ static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; mod build; mod install_many; +mod render_benchmarks; mod resolve_cli; mod resolve_many; mod wheel_metadata; @@ -66,6 +68,7 @@ enum Cli { /// Resolve requirements passed on the CLI Resolve(ResolveCliArgs), WheelMetadata(WheelMetadataArgs), + RenderBenchmarks(RenderBenchmarksArgs), } #[instrument] // Anchor span to check for overhead @@ -86,6 +89,7 @@ async fn run() -> Result<()> { resolve_cli::resolve_cli(args).await?; } Cli::WheelMetadata(args) => wheel_metadata::wheel_metadata(args).await?, + Cli::RenderBenchmarks(args) => render_benchmarks::render_benchmarks(&args)?, } Ok(()) } diff --git a/crates/puffin-dev/src/render_benchmarks.rs b/crates/puffin-dev/src/render_benchmarks.rs new file mode 100644 index 000000000..55ebb4c61 --- /dev/null +++ b/crates/puffin-dev/src/render_benchmarks.rs @@ -0,0 +1,113 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, Result}; +use clap::Parser; +use poloto::build; +use resvg::usvg_text_layout::{fontdb, TreeTextToPath}; +use serde::Deserialize; +use tagu::prelude::*; + +#[derive(Parser)] +pub(crate) struct RenderBenchmarksArgs { + /// Path to a JSON output from a `hyperfine` benchmark. + path: PathBuf, + /// Title of the plot. + #[clap(long, short)] + title: Option, +} + +pub(crate) fn render_benchmarks(args: &RenderBenchmarksArgs) -> Result<()> { + let mut results: BenchmarkResults = serde_json::from_slice(&std::fs::read(&args.path)?)?; + + // Replace the command with a shorter name. (The command typically includes the benchmark name, + // but we assume we're running over a single benchmark here.) + for result in &mut results.results { + if result.command.starts_with("puffin") { + result.command = "puffin".into(); + } else if result.command.starts_with("pip-compile") { + result.command = "pip-compile".into(); + } else if result.command.starts_with("pip-sync") { + result.command = "pip-sync".into(); + } else if result.command.starts_with("poetry") { + result.command = "poetry".into(); + } else { + return Err(anyhow!("unknown command: {}", result.command)); + } + } + + let fontdb = load_fonts(); + + render_to_png( + &plot_benchmark(args.title.as_deref().unwrap_or("Benchmark"), &results)?, + &args.path.with_extension("png"), + &fontdb, + )?; + + Ok(()) +} + +/// Render a benchmark to an SVG (as a string). +fn plot_benchmark(heading: &str, results: &BenchmarkResults) -> Result { + let mut data = Vec::new(); + for result in &results.results { + data.push((result.mean, &result.command)); + } + + let theme = poloto::render::Theme::light(); + let theme = theme.append(tagu::build::raw( + ".poloto0.poloto_fill{fill: #6340AC !important;}", + )); + let theme = theme.append(tagu::build::raw( + ".poloto_background{fill: white !important;}", + )); + + Ok(build::bar::gen_simple("", data, [0.0]) + .label((heading, "Time (s)", "")) + .append_to(poloto::header().append(theme)) + .render_string()?) +} + +/// Render an SVG to a PNG file. +fn render_to_png(data: &str, path: &Path, fontdb: &fontdb::Database) -> Result<()> { + let mut tree = resvg::usvg::Tree::from_str(data, &resvg::usvg::Options::default())?; + tree.convert_text(fontdb); + let fit_to = resvg::usvg::FitTo::Width(1600); + let size = fit_to + .fit_to(tree.size.to_screen_size()) + .ok_or_else(|| anyhow!("failed to fit to screen size"))?; + let mut pixmap = resvg::tiny_skia::Pixmap::new(size.width(), size.height()).unwrap(); + resvg::render( + &tree, + fit_to, + resvg::tiny_skia::Transform::default(), + pixmap.as_mut(), + ) + .ok_or_else(|| anyhow!("failed to render"))?; + std::fs::create_dir_all(path.parent().unwrap())?; + pixmap.save_png(path)?; + Ok(()) +} + +/// Load the system fonts and set the default font families. +fn load_fonts() -> fontdb::Database { + let mut fontdb = fontdb::Database::new(); + fontdb.load_system_fonts(); + fontdb.set_serif_family("Times New Roman"); + fontdb.set_sans_serif_family("Arial"); + fontdb.set_cursive_family("Comic Sans MS"); + fontdb.set_fantasy_family("Impact"); + fontdb.set_monospace_family("Courier New"); + + fontdb +} + +#[derive(Debug, Deserialize)] +struct BenchmarkResults { + results: Vec, +} + +#[derive(Debug, Deserialize)] +struct BenchmarkResult { + command: String, + mean: f64, +} diff --git a/scripts/bench/__main__.py b/scripts/bench/__main__.py index 3b238a4c1..97c92cc95 100644 --- a/scripts/bench/__main__.py +++ b/scripts/bench/__main__.py @@ -59,6 +59,9 @@ class Command(typing.NamedTuple): class Hyperfine(typing.NamedTuple): + benchmark: Benchmark + """The benchmark to run.""" + commands: list[Command] """The commands to benchmark.""" @@ -71,10 +74,18 @@ class Hyperfine(typing.NamedTuple): verbose: bool """Whether to print verbose output.""" + json: bool + """Whether to export results to JSON.""" + def run(self) -> None: """Run the benchmark using `hyperfine`.""" args = ["hyperfine"] + # Export to JSON. + if self.json: + args.append("--export-json") + args.append(f"{self.benchmark.value}.json") + # Preamble: benchmark-wide setup. if self.verbose: args.append("--show-output") @@ -716,6 +727,9 @@ def main(): parser.add_argument( "--verbose", "-v", action="store_true", help="Print verbose output." ) + parser.add_argument( + "--json", action="store_true", help="Export results to JSON." + ) parser.add_argument( "--warmup", type=int, @@ -789,6 +803,7 @@ def main(): ) verbose = args.verbose + json = args.json warmup = args.warmup min_runs = args.min_runs @@ -858,10 +873,12 @@ def main(): if commands: hyperfine = Hyperfine( + benchmark=benchmark, commands=commands, warmup=warmup, min_runs=min_runs, verbose=verbose, + json=json, ) hyperfine.run()