From 3f875a59ba451e6261dac45dabd531f3c8ad77fb Mon Sep 17 00:00:00 2001
From: konstin <konstin@mailbox.org>
Date: Thu, 21 Dec 2023 11:47:37 +0100
Subject: [PATCH] Add pubgrub testing script

I'm using this script to provide feedback to pubgrub changes, and it's better to have it in tree than only on my machine.
---
 scripts/benchmarks/pubgrub.sh              | 50 ++++++++++++++++++++++
 scripts/requirements/bio_embeddings.txt    |  2 +
 scripts/requirements/tf-models-nightly.txt |  2 +
 3 files changed, 54 insertions(+)
 create mode 100644 scripts/benchmarks/pubgrub.sh
 create mode 100644 scripts/requirements/bio_embeddings.txt
 create mode 100644 scripts/requirements/tf-models-nightly.txt

diff --git a/scripts/benchmarks/pubgrub.sh b/scripts/benchmarks/pubgrub.sh
new file mode 100644
index 000000000..377001cd4
--- /dev/null
+++ b/scripts/benchmarks/pubgrub.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+# @konstin's script for providing feedback for pubgrub changes
+
+# No `set -e`, these are error cases
+
+# shellcheck disable=SC2164
+cd "$(git rev-parse --show-toplevel)"
+
+git checkout Cargo.toml
+echo "main: $(cat Cargo.toml | grep pubgrub | sed -e 's/.*rev = "\(.*\)".*/\1/g')"
+echo "branch: $(cd ../pubgrub && git rev-parse HEAD)"
+
+cargo build --profile profiling
+cp target/profiling/puffin target/profiling/main
+cp target/profiling/puffin-dev target/profiling/main-dev
+
+# Patch Cargo.toml
+sed -i 's/pubgrub = .*/pubgrub = { path = "..\/pubgrub" }/g' Cargo.toml
+
+cargo build --profile profiling
+
+mkdir -p target/logs
+
+virtualenv -p 3.12 --clear -q .venv312
+echo "tf-models-nightly main"
+time VIRTUAL_ENV=.venv312 target/profiling/main pip-compile scripts/requirements/tf-models-nightly.txt 2> target/logs/tf-models-nightly-main.txt
+echo "tf-models-nightly branch"
+time VIRTUAL_ENV=.venv312 target/profiling/puffin pip-compile scripts/requirements/tf-models-nightly.txt 2> target/logs/tf-models-nightly-branch.txt
+
+# I don't understand why this doesn't work, the flamegraphs either never finish (https://michcioperz.com/wiki/slow-perf-script/)
+# or they are unusable compared to sample record, which doesn't seem to have this problem at all but can't export svg flamegraphs
+# https://github.com/flamegraph-rs/flamegraph/pull/127
+# VIRTUAL_ENV=.venv312 perf record --call-graph dwarf target/profiling/main pip-compile scripts/requirements/tf-models-nightly.txt 2> /dev/null
+# flamegraph --perfdata perf.data --no-inline -o flamegraph-tf-models-nightly-main.svg
+# VIRTUAL_ENV=.venv312 perf record --call-graph dwarf target/profiling/branch pip-compile scripts/requirements/tf-models-nightly.txt 2> /dev/null
+# flamegraph --perfdata perf.data --no-inline -o flamegraph-tf-models-nightly-branch.svg --open
+
+virtualenv -p 3.10 --clear -q .venv310
+echo "bio_embeddings main"
+time VIRTUAL_ENV=.venv310 target/profiling/main pip-compile scripts/requirements/bio_embeddings.txt 2> target/logs/bio_embeddings-main.txt > /dev/null
+echo "bio_embeddings branch"
+time VIRTUAL_ENV=.venv310 target/profiling/puffin pip-compile scripts/requirements/bio_embeddings.txt 2> target/logs/bio_embeddings-branch.txt > /dev/null
+
+virtualenv -p 3.12 --clear -q .venv312
+echo "boto3 main"
+time VIRTUAL_ENV=.venv312 target/profiling/main pip-compile scripts/requirements/boto3.txt 2> target/logs/boto3-main.txt > /dev/null
+echo "boto3 branch"
+time VIRTUAL_ENV=.venv312 target/profiling/puffin pip-compile scripts/requirements/boto3.txt 2> target/logs/boto3-branch.txt > /dev/null
+
+git checkout Cargo.toml
diff --git a/scripts/requirements/bio_embeddings.txt b/scripts/requirements/bio_embeddings.txt
new file mode 100644
index 000000000..03e82451c
--- /dev/null
+++ b/scripts/requirements/bio_embeddings.txt
@@ -0,0 +1,2 @@
+# Run this on python 3.10 for a pathologically slow case of backtracking through many versions due to botocore and urllib
+bio-embeddings
diff --git a/scripts/requirements/tf-models-nightly.txt b/scripts/requirements/tf-models-nightly.txt
new file mode 100644
index 000000000..8b163a406
--- /dev/null
+++ b/scripts/requirements/tf-models-nightly.txt
@@ -0,0 +1,2 @@
+# Run this on python 3.12 for a pathologically slow case of backtracking through many versions
+tf-models-nightly