mirror of https://github.com/mongodb/mongo
SERVER-112016 DRY and make bazel CI scripts restart on OOM (#42300)
GitOrigin-RevId: 6ffb7cd551d8ba59ab93fdeb7b8c4f74fcf2cea8
This commit is contained in:
Daniel Moody
MongoDB Bot
parent
fd9c122a0d
commit
612b418551
|
|
@ -16,11 +16,6 @@ sh_binary(
|
|||
srcs = ["antithesis_image_build_and_push.sh"],
|
||||
)
|
||||
|
||||
sh_binary(
|
||||
name = "bazel_RBE_supported",
|
||||
srcs = ["bazel_RBE_supported.sh"],
|
||||
)
|
||||
|
||||
sh_binary(
|
||||
name = "bazel_compile",
|
||||
srcs = ["bazel_compile.sh"],
|
||||
|
|
@ -31,24 +26,19 @@ sh_binary(
|
|||
srcs = ["bazel_coverage.sh"],
|
||||
)
|
||||
|
||||
sh_binary(
|
||||
name = "bazel_get_binary_path",
|
||||
srcs = ["bazel_get_binary_path.sh"],
|
||||
)
|
||||
|
||||
sh_binary(
|
||||
name = "bazel_run",
|
||||
srcs = ["bazel_run.sh"],
|
||||
)
|
||||
|
||||
sh_binary(
|
||||
name = "bazel_scons_diff",
|
||||
srcs = ["bazel_scons_diff.sh"],
|
||||
name = "bazel_test",
|
||||
srcs = ["bazel_test.sh"],
|
||||
)
|
||||
|
||||
sh_binary(
|
||||
name = "bazel_utility_functions",
|
||||
srcs = ["bazel_utility_functions.sh"],
|
||||
name = "bazel_evergreen_shutils",
|
||||
srcs = ["bazel_evergreen_shutils.sh"],
|
||||
)
|
||||
|
||||
sh_binary(
|
||||
|
|
@ -76,11 +66,6 @@ sh_binary(
|
|||
srcs = ["cleanup_environment.sh"],
|
||||
)
|
||||
|
||||
sh_binary(
|
||||
name = "compile_ninja",
|
||||
srcs = ["compile_ninja.sh"],
|
||||
)
|
||||
|
||||
sh_binary(
|
||||
name = "compiled_binaries_get",
|
||||
srcs = ["compiled_binaries_get.sh"],
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
|
||||
|
||||
source "$DIR/bazel_utility_functions.sh"
|
||||
(
|
||||
cd $DIR/..
|
||||
exec $(bazel_get_binary_path) "$@"
|
||||
)
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
bazel_rbe_supported() {
|
||||
|
||||
OS="$(uname)"
|
||||
ARCH="$(uname -m)"
|
||||
|
||||
if [ "$ARCH" == "aarch64" ] || [ "$ARCH" == "arm64" ] || [ "$ARCH" == "x86_64" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
|
@ -11,57 +11,49 @@
|
|||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
|
||||
. "$DIR/prelude.sh"
|
||||
|
||||
cd src
|
||||
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
set -o pipefail
|
||||
|
||||
activate_venv
|
||||
. "$DIR/bazel_evergreen_shutils.sh"
|
||||
|
||||
bazel_evergreen_shutils::activate_and_cd_src
|
||||
bazel_evergreen_shutils::export_ssl_paths_if_needed
|
||||
|
||||
# if build_patch_id is passed, try to download binaries from specified
|
||||
# evergreen patch.
|
||||
build_patch_id="${build_patch_id:-${reuse_compile_from}}"
|
||||
if [ -n "${build_patch_id}" ]; then
|
||||
if [[ -n "${build_patch_id}" ]]; then
|
||||
echo "build_patch_id detected, trying to skip task"
|
||||
|
||||
# On windows we change the extension to zip
|
||||
if [ -z "${ext}" ]; then
|
||||
ext="tgz"
|
||||
fi
|
||||
|
||||
if [[ -z "${ext:-}" ]]; then ext="tgz"; fi
|
||||
extra_db_contrib_args=""
|
||||
|
||||
# get the platform of the dist archive. This is needed if
|
||||
# db-contrib-tool cannot autodetect the platform of the ec2 instance.
|
||||
regex='MONGO_DISTMOD=([a-z0-9]*)'
|
||||
if [[ ${bazel_compile_flags} =~ ${regex} ]]; then
|
||||
if [[ ${bazel_compile_flags:-} =~ ${regex} ]]; then
|
||||
extra_db_contrib_args="${extra_db_contrib_args} --platform=${BASH_REMATCH[1]}"
|
||||
fi
|
||||
|
||||
download_dir="./tmp_db_contrib_tool_download_dir"
|
||||
rm -rf ${download_dir}
|
||||
rm -rf "${download_dir}"
|
||||
|
||||
if [ "${task_name}" = "archive_dist_test" ]; then
|
||||
invocation=""
|
||||
if [[ "${task_name:-}" == "archive_dist_test" ]]; then
|
||||
file_name="dist-test-stripped.${ext}"
|
||||
invocation="db-contrib-tool setup-repro-env ${build_patch_id} \
|
||||
--variant=${compile_variant} --extractDownloads=False \
|
||||
--binariesName=${file_name} --installDir=${download_dir} ${extra_db_contrib_args}"
|
||||
invocation="db-contrib-tool setup-repro-env ${build_patch_id} --variant=${compile_variant} --extractDownloads=False --binariesName=${file_name} --installDir=${download_dir} ${extra_db_contrib_args}"
|
||||
fi
|
||||
|
||||
if [ "${task_name}" = "archive_dist_test_debug" ]; then
|
||||
if [[ "${task_name:-}" == "archive_dist_test_debug" ]]; then
|
||||
file_name="dist-test-debug.${ext}"
|
||||
invocation="db-contrib-tool setup-repro-env ${build_patch_id} \
|
||||
--variant=${compile_variant} --extractDownloads=False \
|
||||
--debugsymbolsName=${file_name} --installDir=${download_dir} \
|
||||
--skipBinaries --downloadSymbols ${extra_db_contrib_args}"
|
||||
invocation="db-contrib-tool setup-repro-env ${build_patch_id} --variant=${compile_variant} --extractDownloads=False --debugsymbolsName=${file_name} --installDir=${download_dir} --skipBinaries --downloadSymbols ${extra_db_contrib_args}"
|
||||
fi
|
||||
|
||||
if [ -n "${invocation}" ]; then
|
||||
if [[ -n "${invocation}" ]]; then
|
||||
setup_db_contrib_tool
|
||||
|
||||
echo "db-contrib-tool invocation: ${invocation}"
|
||||
eval ${invocation}
|
||||
if [ $? -ne 0 ]; then
|
||||
if ! eval ${invocation}; then
|
||||
echo "Could not retrieve files with db-contrib-tool"
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -79,10 +71,10 @@ fi
|
|||
|
||||
# --build-mongot is a compile flag used by the evergreen build variants that run end-to-end search
|
||||
# suites, as it downloads the necessary mongot binary.
|
||||
if [ "${build_mongot}" = "true" ]; then
|
||||
if [[ "${build_mongot:-}" == "true" ]]; then
|
||||
setup_db_contrib_tool
|
||||
use_db_contrib_tool_mongot
|
||||
bazel_args="${bazel_args} --include_mongot=True"
|
||||
bazel_args="${bazel_args:-} --include_mongot=True"
|
||||
fi
|
||||
|
||||
# This is hacky way to pass off build time from archive_dist_test to archive_dist_test_debug
|
||||
|
|
@ -90,12 +82,12 @@ fi
|
|||
# We then create the symbols normally in archive_dist_test_debug. We have to force the
|
||||
# build-id for debugging as they will be different when -Wl,-S is passed in.
|
||||
# The relinked binaries should still be hash identical when stripped with strip
|
||||
if [ "${skip_debug_link}" = "true" ]; then
|
||||
if [[ "${skip_debug_link:-}" == "true" ]]; then
|
||||
export compile_variant="${compile_variant}"
|
||||
export version_id="${version_id}"
|
||||
if [ "${task_name}" = "archive_dist_test" ]; then
|
||||
task_compile_flags="${task_compile_flags} --simple_build_id=True --features=strip_debug --separate_debug=False"
|
||||
if [ "${remote_link}" = "true" ]; then
|
||||
if [[ "${task_name:-}" == "archive_dist_test" ]]; then
|
||||
task_compile_flags="${task_compile_flags:-} --simple_build_id=True --features=strip_debug --separate_debug=False"
|
||||
if [[ "${remote_link:-}" == "true" ]]; then
|
||||
ARCH=$(uname -m)
|
||||
# Remote linking is currently only supported on arm
|
||||
if [[ "$ARCH" == "arm64" || "$ARCH" == "aarch64" ]]; then
|
||||
|
|
@ -103,66 +95,45 @@ if [ "${skip_debug_link}" = "true" ]; then
|
|||
fi
|
||||
fi
|
||||
fi
|
||||
if [ "${task_name}" = "archive_dist_test_debug" ]; then
|
||||
task_compile_flags="${task_compile_flags} --simple_build_id=True"
|
||||
if [[ "${task_name:-}" == "archive_dist_test_debug" ]]; then
|
||||
task_compile_flags="${task_compile_flags:-} --simple_build_id=True"
|
||||
fi
|
||||
fi
|
||||
|
||||
set -o pipefail
|
||||
|
||||
# Use `eval` to force evaluation of the environment variables in the echo statement:
|
||||
eval echo "Execution environment: Targets: ${targets}"
|
||||
|
||||
source ./evergreen/bazel_utility_functions.sh
|
||||
source ./evergreen/bazel_RBE_supported.sh
|
||||
BAZEL_BINARY="$(bazel_evergreen_shutils::bazel_get_binary_path)"
|
||||
|
||||
if [[ "${evergreen_remote_exec}" != "on" ]]; then
|
||||
LOCAL_ARG="$LOCAL_ARG --jobs=auto"
|
||||
fi
|
||||
|
||||
BAZEL_BINARY=$(bazel_get_binary_path)
|
||||
|
||||
# Timeout is set here to avoid the build hanging indefinitely, still allowing
|
||||
# for retries.
|
||||
TIMEOUT_CMD=""
|
||||
if [ -n "${build_timeout_seconds}" ]; then
|
||||
TIMEOUT_CMD="timeout ${build_timeout_seconds}"
|
||||
elif [[ "${evergreen_remote_exec}" == "on" ]]; then
|
||||
# Timeout remote execution runs in 60 minutes as a workaround for
|
||||
# scheduling timeout bugs
|
||||
TIMEOUT_CMD="timeout 3600"
|
||||
fi
|
||||
|
||||
if is_ppc64le; then
|
||||
LOCAL_ARG="$LOCAL_ARG --jobs=48"
|
||||
fi
|
||||
|
||||
if is_s390x; then
|
||||
LOCAL_ARG="$LOCAL_ARG --jobs=16"
|
||||
fi
|
||||
# Compose LOCAL_ARG and release flags
|
||||
LOCAL_ARG="$(bazel_evergreen_shutils::compute_local_arg build)"
|
||||
|
||||
# If we are doing a patch build or we are building a non-push
|
||||
# build on the waterfall, then we don't need the --release
|
||||
# flag. Otherwise, this is potentially a build that "leaves
|
||||
# the building", so we do want that flag.
|
||||
if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then
|
||||
echo "This is a non-release build."
|
||||
else
|
||||
LOCAL_ARG="$LOCAL_ARG --config=public-release"
|
||||
LOCAL_ARG="$(bazel_evergreen_shutils::maybe_release_flag "$LOCAL_ARG")"
|
||||
|
||||
# Ensure server is up and print PID
|
||||
bazel_evergreen_shutils::ensure_server_and_print_pid "$BAZEL_BINARY"
|
||||
|
||||
# Build flags line
|
||||
ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${bazel_args:-} ${bazel_compile_flags:-} ${task_compile_flags:-} --define=MONGO_VERSION=${version} ${patch_compile_flags:-}"
|
||||
echo "${ALL_FLAGS}" >.bazel_build_flags
|
||||
|
||||
set +o errexit
|
||||
|
||||
bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \
|
||||
build ${ALL_FLAGS} ${targets}
|
||||
RET=$?
|
||||
|
||||
set -o errexit
|
||||
|
||||
if [[ "$RET" -eq 124 ]]; then
|
||||
echo "Bazel build timed out after ${build_timeout_seconds:-<unspecified>} seconds."
|
||||
elif [[ "$RET" != "0" ]]; then
|
||||
echo "Bazel build failed after retries."
|
||||
fi
|
||||
|
||||
for i in {1..3}; do
|
||||
eval ${TIMEOUT_CMD} $BAZEL_BINARY build --verbose_failures $LOCAL_ARG ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \
|
||||
--define=MONGO_VERSION=${version} ${patch_compile_flags} ${targets} 2>&1 | tee bazel_stdout.log &&
|
||||
RET=0 && break || RET=$? && sleep 60
|
||||
if [ $RET -eq 124 ]; then
|
||||
echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..."
|
||||
else
|
||||
echo "Errors were found during the bazel run, here are the errors:" 1>&2
|
||||
grep "ERROR:" bazel_stdout.log 1>&2
|
||||
echo "Bazel failed to execute, retrying..."
|
||||
fi
|
||||
$BAZEL_BINARY shutdown
|
||||
done
|
||||
|
||||
exit $RET
|
||||
: "${RET:=1}"
|
||||
exit "${RET}"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,335 @@
|
|||
# Common helpers for CI Bazel scripts (build/run/test)
|
||||
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
# --- Pre-flight (assumes prelude.sh already sourced by caller) -------------
|
||||
|
||||
bazel_evergreen_shutils::activate_and_cd_src() {
|
||||
cd src
|
||||
set -o verbose
|
||||
activate_venv
|
||||
}
|
||||
|
||||
# --- Distro quirks -----------------------------------------------------
|
||||
|
||||
bazel_evergreen_shutils::export_ssl_paths_if_needed() {
|
||||
if [[ -f /etc/os-release ]]; then
|
||||
local DISTRO
|
||||
DISTRO=$(awk -F '[="]*' '/^PRETTY_NAME/ { print $2 }' </etc/os-release)
|
||||
if [[ "$DISTRO" == "Amazon Linux 2" ]]; then
|
||||
export SSL_CERT_DIR=/etc/pki/tls/certs
|
||||
export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
|
||||
elif [[ "$DISTRO" == "Red Hat Enterprise Linux"* ]]; then
|
||||
export SSL_CERT_DIR=/etc/pki/ca-trust/extracted/pem
|
||||
export SSL_CERT_FILE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
bazel_evergreen_shutils::is_macos() {
|
||||
local -r os="$(uname -s | tr '[:upper:]' '[:lower:]')"
|
||||
[[ "${os}" == "darwin" ]] && return 0 || return 1
|
||||
}
|
||||
|
||||
bazel_evergreen_shutils::is_ppc64le() {
|
||||
local -r arch="$(uname -m)"
|
||||
[[ "${arch}" == "ppc64le" || "${arch}" == "ppc64" || "${arch}" == "ppc" ]] && return 0 || return 1
|
||||
}
|
||||
|
||||
bazel_evergreen_shutils::is_s390x() {
|
||||
local -r arch="$(uname -m)"
|
||||
[[ "${arch}" == "s390x" || "${arch}" == "s390" ]] && return 0 || return 1
|
||||
}
|
||||
|
||||
bazel_evergreen_shutils::is_s390x_or_ppc64le() {
|
||||
if bazel_evergreen_shutils::is_ppc64le || bazel_evergreen_shutils::is_s390x; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
bazel_evergreen_shutils::bazel_get_binary_path() {
|
||||
if bazel_evergreen_shutils::is_macos; then
|
||||
echo "bazel"
|
||||
elif bazel_evergreen_shutils::is_s390x_or_ppc64le ||
|
||||
grep -q "ID=debian" /etc/os-release ||
|
||||
grep -q 'ID="sles"' /etc/os-release; then
|
||||
echo "bazel/bazelisk.py"
|
||||
else
|
||||
echo "bazel"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- RBE/local flags + arch tuning ----------------------------------------
|
||||
|
||||
bazel_evergreen_shutils::bazel_rbe_supported() {
|
||||
|
||||
local OS ARCH
|
||||
OS="$(uname)"
|
||||
ARCH="$(uname -m)"
|
||||
|
||||
if [ "$ARCH" == "aarch64" ] || [ "$ARCH" == "arm64" ] || [ "$ARCH" == "x86_64" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Requires: evergreen_remote_exec, task_name (for tests), bazel_args vars optionally.
|
||||
bazel_evergreen_shutils::compute_local_arg() {
|
||||
local mode="${1:-build}" # build|test|run
|
||||
local local_arg=""
|
||||
if [[ "${evergreen_remote_exec:-}" != "on" ]]; then
|
||||
local_arg+=" --jobs=auto"
|
||||
elif [[ "$mode" == "test" && "${task_name:-}" == "unit_tests" ]]; then
|
||||
local_arg+=" --config=remote_test"
|
||||
fi
|
||||
|
||||
if bazel_evergreen_shutils::is_ppc64le; then
|
||||
local_arg+=" --jobs=48"
|
||||
fi
|
||||
if bazel_evergreen_shutils::is_s390x; then
|
||||
local_arg+=" --jobs=16"
|
||||
fi
|
||||
|
||||
# For run-mode, if RBE isn't supported or is disabled explicitly, force local config.
|
||||
if [[ "$mode" == "run" ]]; then
|
||||
if ! bazel_evergreen_shutils::bazel_rbe_supported || [[ "${evergreen_remote_exec:-}" != "on" ]]; then
|
||||
# Keep compatibility with existing pattern:
|
||||
local_arg+=" --config=local"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "$local_arg"
|
||||
}
|
||||
|
||||
# Keeps only --config flags from a flag string (used to persist consistency)
|
||||
bazel_evergreen_shutils::extract_config_flags() {
|
||||
local all="$*"
|
||||
awk '{
|
||||
for (i=1;i<=NF;i++) if ($i ~ /^--config(=|$)/) printf "%s ", $i
|
||||
}' <<<"$all"
|
||||
}
|
||||
|
||||
# Adds --config=public-release if this is a release-ish build.
|
||||
bazel_evergreen_shutils::maybe_release_flag() {
|
||||
local local_arg="$1"
|
||||
if [[ "${is_patch:-}" == "true" || -z "${push_bucket:-}" || "${compiling_for_test:-}" == "true" ]]; then
|
||||
echo "$local_arg" # non-release
|
||||
else
|
||||
echo "$local_arg --config=public-release"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Timeouts --------------------------------------------------------------
|
||||
|
||||
# Timeout helper: returns a "timeout <secs>" prefix or empty string.
|
||||
# Prints a one-time warning to stderr if a timeout was requested but no
|
||||
# timeout binary is available. Supports macOS 'gtimeout' if installed.
|
||||
bazel_evergreen_shutils::timeout_prefix() {
|
||||
local fallback_remote="${1:-}" # "on" = use 3600s default for remote builds
|
||||
local need_timeout="" # "explicit" | "fallback" | ""
|
||||
local timeout_bin=""
|
||||
|
||||
# Do we want a timeout?
|
||||
if [[ -n "${build_timeout_seconds:-}" ]]; then
|
||||
need_timeout="explicit"
|
||||
elif [[ "$fallback_remote" == "on" ]]; then
|
||||
need_timeout="fallback"
|
||||
fi
|
||||
|
||||
# Find a timeout binary (GNU coreutils 'timeout' or macOS 'gtimeout')
|
||||
if command -v timeout >/dev/null 2>&1; then
|
||||
timeout_bin="timeout"
|
||||
elif command -v gtimeout >/dev/null 2>&1; then
|
||||
timeout_bin="gtimeout"
|
||||
fi
|
||||
|
||||
# If needed but unavailable, warn once and return empty
|
||||
if [[ -n "$need_timeout" && -z "$timeout_bin" ]]; then
|
||||
if [[ -z "${_BAZEL_EVG_TIMEOUT_WARNED:-}" ]]; then
|
||||
if [[ "$need_timeout" == "explicit" ]]; then
|
||||
echo "[warn] 'timeout' not found; requested ${build_timeout_seconds}s timeout will be ignored." >&2
|
||||
else
|
||||
echo "[warn] 'timeout' not found; remote-build fallback timeout (3600s) will be ignored." >&2
|
||||
fi
|
||||
# Helpful hint for macOS users
|
||||
if bazel_evergreen_shutils::is_macos; then
|
||||
echo "[hint] On macOS, install coreutils: 'brew install coreutils' (provides 'gtimeout')." >&2
|
||||
fi
|
||||
_BAZEL_EVG_TIMEOUT_WARNED=1
|
||||
fi
|
||||
echo "" # no timeout prefix
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Produce the prefix if we have a binary
|
||||
if [[ -n "$timeout_bin" ]]; then
|
||||
if [[ "$need_timeout" == "explicit" ]]; then
|
||||
echo "$timeout_bin ${build_timeout_seconds}"
|
||||
elif [[ "$need_timeout" == "fallback" ]]; then
|
||||
echo "$timeout_bin 3600"
|
||||
else
|
||||
echo ""
|
||||
fi
|
||||
else
|
||||
echo ""
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Bazel server lifecycle & OOM-detect retry -----------------------------
|
||||
|
||||
bazel_evergreen_shutils::bazel_output_base() {
|
||||
local BAZEL_BINARY="$1"
|
||||
"$BAZEL_BINARY" info output_base 2>/dev/null
|
||||
}
|
||||
|
||||
bazel_evergreen_shutils::bazel_pidfile_path() {
|
||||
local BAZEL_BINARY="$1"
|
||||
local ob
|
||||
ob="$(bazel_evergreen_shutils::bazel_output_base "$BAZEL_BINARY")" || return 1
|
||||
echo "${ob}/server/server.pid.txt"
|
||||
}
|
||||
|
||||
bazel_evergreen_shutils::is_bazel_server_running() {
|
||||
local BAZEL_BINARY="$1"
|
||||
local pf pid
|
||||
pf="$(bazel_evergreen_shutils::bazel_pidfile_path "$BAZEL_BINARY")" || return 1
|
||||
[[ -f "$pf" ]] || return 1
|
||||
pid="$(cat "$pf" 2>/dev/null || true)"
|
||||
[[ -n "$pid" ]] || return 1
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
bazel_evergreen_shutils::print_bazel_server_pid() {
|
||||
local BAZEL_BINARY="$1"
|
||||
local pf pid
|
||||
pf="$(bazel_evergreen_shutils::bazel_pidfile_path "$BAZEL_BINARY")" || {
|
||||
echo "Bazel server pidfile not found (output_base: $(bazel_evergreen_shutils::bazel_output_base "$BAZEL_BINARY" 2>/dev/null || true))"
|
||||
return 0
|
||||
}
|
||||
if [[ -f "$pf" ]]; then
|
||||
pid="$(cat "$pf" 2>/dev/null || true)"
|
||||
echo "Bazel server pidfile: $pf (PID=${pid:-unknown})"
|
||||
else
|
||||
echo "Bazel server pidfile not found yet (output_base: $(bazel_evergreen_shutils::bazel_output_base "$BAZEL_BINARY" 2>/dev/null || true))"
|
||||
fi
|
||||
}
|
||||
|
||||
# Starts server (if needed) and prints PID. Safe to call multiple times.
|
||||
bazel_evergreen_shutils::ensure_server_and_print_pid() {
|
||||
local BAZEL_BINARY="$1"
|
||||
_IGN=$("$BAZEL_BINARY" info >/dev/null 2>&1 || true)
|
||||
bazel_evergreen_shutils::print_bazel_server_pid "$BAZEL_BINARY"
|
||||
}
|
||||
|
||||
# Generic retry wrapper:
|
||||
# $1: attempts
|
||||
# $3: bazel binary
|
||||
# $4..: full bazel subcommand + args (e.g. "build --verbose_failures ...")
|
||||
# Special handling:
|
||||
# - exit 124 -> timeout
|
||||
# - server death (pid missing) -> restart, then retry
|
||||
# Returns with global RET set.
|
||||
bazel_evergreen_shutils::retry_bazel_cmd() {
|
||||
local attempts="$1"
|
||||
shift
|
||||
local BAZEL_BINARY="$1"
|
||||
shift
|
||||
|
||||
local timeout_str="$(bazel_evergreen_shutils::timeout_prefix "${evergreen_remote_exec:-}")"
|
||||
|
||||
# Everything else is the Bazel subcommand + flags (and possibly redirections/pipes).
|
||||
# We *intentionally* keep it as raw words and reassemble to a single string for eval.
|
||||
local raw_rest=("$@")
|
||||
|
||||
# Once we detect an OOM/server-death, we enable the guard for subsequent attempts.
|
||||
local use_oom_guard=false
|
||||
local -r OOM_GUARD_FLAG='--local_resources=HOST_CPUS*.5'
|
||||
|
||||
# Helper: does the current command string already include a local_resources flag?
|
||||
_cmd_has_local_resources() {
|
||||
[[ "$1" == *"--local_resources"* ]]
|
||||
}
|
||||
|
||||
local RET=1
|
||||
|
||||
for i in $(seq 1 "$attempts"); do
|
||||
echo "Attempt ${i}/${attempts}…" >&2
|
||||
|
||||
# Ensure/refresh server & pid before we run (helps produce a fresh pidfile too).
|
||||
if ! bazel_evergreen_shutils::is_bazel_server_running "$BAZEL_BINARY"; then
|
||||
echo "[retry ${i}] Bazel server not running (likely OOM/killed); restarting…" >&2
|
||||
"$BAZEL_BINARY" info >/dev/null 2>&1 || true
|
||||
bazel_evergreen_shutils::print_bazel_server_pid "$BAZEL_BINARY" >&2
|
||||
fi
|
||||
|
||||
# Reassemble the caller’s words into a single command string for eval.
|
||||
# We deliberately do *not* try to be clever here—this restores legacy behavior
|
||||
# where quoted pieces inside variables (e.g., --base_dir="..") are honored by the shell.
|
||||
local cmd="\"$BAZEL_BINARY\""
|
||||
local w
|
||||
for w in "${raw_rest[@]}"; do
|
||||
cmd+=" $w"
|
||||
done
|
||||
|
||||
# If OOM guard is enabled and not already present, append it.
|
||||
# (Safe with eval; if the caller added redirections earlier, this is still just an arg.)
|
||||
if $use_oom_guard && ! _cmd_has_local_resources "$cmd"; then
|
||||
echo "[retry ${i}] Applying OOM guard: ${OOM_GUARD_FLAG}" >&2
|
||||
cmd+=" ${OOM_GUARD_FLAG}"
|
||||
fi
|
||||
|
||||
# Prefix timeout, if any.
|
||||
if [[ -n "$timeout_str" ]]; then
|
||||
cmd="${timeout_str} ${cmd}"
|
||||
fi
|
||||
|
||||
# Run it.
|
||||
# NOTE: We *do not* add any redirections here; caller controls logging completely.
|
||||
if eval "$cmd"; then
|
||||
RET=0
|
||||
break
|
||||
else
|
||||
RET=$?
|
||||
fi
|
||||
|
||||
# Classify failure & decide on guard for next attempt.
|
||||
[[ $RET -eq 124 ]] && echo "Bazel timed out." >&2
|
||||
|
||||
if ! bazel_evergreen_shutils::is_bazel_server_running "$BAZEL_BINARY"; then
|
||||
echo "[retry ${i}] Bazel server down (OOM/killed). Enabling OOM guard for next attempt and restarting…" >&2
|
||||
use_oom_guard=true
|
||||
"$BAZEL_BINARY" shutdown || true
|
||||
"$BAZEL_BINARY" info >/dev/null 2>&1 || true
|
||||
bazel_evergreen_shutils::print_bazel_server_pid "$BAZEL_BINARY" >&2
|
||||
else
|
||||
echo "Bazel failed (exit=$RET); restarting server before retry…" >&2
|
||||
"$BAZEL_BINARY" shutdown || true
|
||||
fi
|
||||
|
||||
sleep 60
|
||||
done
|
||||
|
||||
return "$RET"
|
||||
|
||||
}
|
||||
|
||||
# --- Test helpers ----------------------------------------------------------
|
||||
|
||||
# Multiplies test timeout on slow arches and appends to bazel_args
|
||||
bazel_evergreen_shutils::maybe_scale_test_timeout_and_append() {
|
||||
if [[ -n "${test_timeout_sec:-}" ]]; then
|
||||
local scaled="$test_timeout_sec"
|
||||
if bazel_evergreen_shutils::is_s390x_or_ppc64le; then
|
||||
scaled=$((test_timeout_sec * 4))
|
||||
fi
|
||||
bazel_args="${bazel_args:-} --test_timeout=${scaled}"
|
||||
fi
|
||||
}
|
||||
|
|
@ -7,73 +7,54 @@
|
|||
# * ${env} - Env variable string to set (ex. ENV_VAR_ABC=123)
|
||||
# * ${redact_args} - If set, redact the args in the report
|
||||
|
||||
# Needed for evergreen scripts that use evergreen expansions and utility methods.
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
|
||||
. "$DIR/prelude.sh"
|
||||
|
||||
cd src
|
||||
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
set -o pipefail
|
||||
|
||||
activate_venv
|
||||
. "$DIR/bazel_evergreen_shutils.sh"
|
||||
|
||||
bazel_evergreen_shutils::activate_and_cd_src
|
||||
bazel_evergreen_shutils::export_ssl_paths_if_needed
|
||||
|
||||
# Use `eval` to force evaluation of the environment variables in the echo statement:
|
||||
eval echo "Execution environment: Args: ${args} Target: ${target} Env: ${env} redact_args: ${redact_args}"
|
||||
|
||||
source ./evergreen/bazel_utility_functions.sh
|
||||
source ./evergreen/bazel_RBE_supported.sh
|
||||
BAZEL_BINARY="$(bazel_evergreen_shutils::bazel_get_binary_path)"
|
||||
|
||||
if bazel_rbe_supported; then
|
||||
LOCAL_ARG=""
|
||||
else
|
||||
LOCAL_ARG="--config=local"
|
||||
fi
|
||||
|
||||
if [[ "${evergreen_remote_exec}" != "on" ]]; then
|
||||
LOCAL_ARG="--config=local"
|
||||
fi
|
||||
|
||||
BAZEL_BINARY=$(bazel_get_binary_path)
|
||||
|
||||
# AL2 stores certs in a nonstandard location
|
||||
if [[ -f /etc/os-release ]]; then
|
||||
DISTRO=$(awk -F '[="]*' '/^PRETTY_NAME/ { print $2 }' </etc/os-release)
|
||||
if [[ $DISTRO == "Amazon Linux 2" ]]; then
|
||||
export SSL_CERT_DIR=/etc/pki/tls/certs
|
||||
export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
|
||||
elif [[ $DISTRO == "Red Hat Enterprise Linux"* ]]; then
|
||||
export SSL_CERT_DIR=/etc/pki/ca-trust/extracted/pem
|
||||
export SSL_CERT_FILE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -n "$redact_args" ]]; then
|
||||
INVOCATION_WITH_REDACTION="${target}"
|
||||
else
|
||||
INVOCATION_WITH_REDACTION="${target} ${args}"
|
||||
fi
|
||||
|
||||
# The --config flag needs to stay consistent between invocations to avoid evicting the previous results.
|
||||
# Strip out anything that isn't a --config flag that could interfere with the run command.
|
||||
# Build LOCAL_ARG for run-mode
|
||||
LOCAL_ARG="$(bazel_evergreen_shutils::compute_local_arg run)"
|
||||
# Honor .bazel_build_flags --config lines (do not evict previous cache)
|
||||
ALL_FLAGS=""
|
||||
if [[ -f .bazel_build_flags ]]; then
|
||||
ALL_FLAGS=$(<.bazel_build_flags)
|
||||
ALL_FLAGS="$(<.bazel_build_flags)"
|
||||
fi
|
||||
CONFIG_FLAGS=$(echo "${ALL_FLAGS}" | tr ' ' '\n' | grep -- '--config' | tr '\n' ' ')
|
||||
|
||||
CONFIG_FLAGS="$(bazel_evergreen_shutils::extract_config_flags "${ALL_FLAGS}")"
|
||||
LOCAL_ARG="${CONFIG_FLAGS} ${LOCAL_ARG}"
|
||||
|
||||
# Print command being run to file that can be uploaded
|
||||
echo "bazel run --verbose_failures ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >>bazel-invocation.txt
|
||||
INVOCATION_WITH_REDACTION="${target}"
|
||||
if [[ -z "${redact_args:-}" ]]; then
|
||||
INVOCATION_WITH_REDACTION+=" ${args}"
|
||||
fi
|
||||
|
||||
# Run bazel command, retrying up to five times
|
||||
MAX_ATTEMPTS=5
|
||||
for ((i = 1; i <= $MAX_ATTEMPTS; i++)); do
|
||||
eval $env $BAZEL_BINARY run --verbose_failures ${LOCAL_ARG} ${target} ${args} >>bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 10
|
||||
if [ $i -lt $MAX_ATTEMPTS ]; then echo "Bazel failed to execute, retrying ($(($i + 1)) of $MAX_ATTEMPTS attempts)... " >>bazel_output.log 2>&1; fi
|
||||
$BAZEL_BINARY shutdown
|
||||
done
|
||||
# Record invocation
|
||||
echo "bazel run --verbose_failures ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >bazel-invocation.txt
|
||||
|
||||
$python ./buildscripts/simple_report.py --test-name "bazel run ${INVOCATION_WITH_REDACTION}" --log-file bazel_output.log --exit-code $RET
|
||||
exit $RET
|
||||
# capture exit code
|
||||
set +o errexit
|
||||
|
||||
bazel_evergreen_shutils::retry_bazel_cmd 5 "$BAZEL_BINARY" \
|
||||
run --verbose_failures ${LOCAL_ARG} ${target} ${args} 2>&1 | tee -a bazel_output.log
|
||||
RET=${PIPESTATUS[0]}
|
||||
: "${RET:=1}"
|
||||
set -o errexit
|
||||
|
||||
# Report
|
||||
$python ./buildscripts/simple_report.py \
|
||||
--test-name "bazel run ${INVOCATION_WITH_REDACTION}" \
|
||||
--log-file bazel_output.log \
|
||||
--exit-code "${RET}"
|
||||
|
||||
exit "${RET}"
|
||||
|
|
|
|||
|
|
@ -8,99 +8,65 @@
|
|||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
|
||||
. "$DIR/prelude.sh"
|
||||
|
||||
cd src
|
||||
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
activate_venv
|
||||
|
||||
set -o pipefail
|
||||
|
||||
. "$DIR/bazel_evergreen_shutils.sh"
|
||||
|
||||
bazel_evergreen_shutils::activate_and_cd_src
|
||||
bazel_evergreen_shutils::export_ssl_paths_if_needed
|
||||
|
||||
# Use `eval` to force evaluation of the environment variables in the echo statement:
|
||||
eval echo "Execution environment: Targets: ${targets}"
|
||||
|
||||
source ./evergreen/bazel_utility_functions.sh
|
||||
source ./evergreen/bazel_RBE_supported.sh
|
||||
BAZEL_BINARY="$(bazel_evergreen_shutils::bazel_get_binary_path)"
|
||||
|
||||
LOCAL_ARG=""
|
||||
if [[ "${evergreen_remote_exec}" != "on" ]]; then
|
||||
LOCAL_ARG="$LOCAL_ARG --jobs=auto"
|
||||
elif [[ "${task_name}" == "unit_tests" ]]; then
|
||||
LOCAL_ARG="$LOCAL_ARG --config=remote_test"
|
||||
fi
|
||||
|
||||
BAZEL_BINARY=$(bazel_get_binary_path)
|
||||
|
||||
# Timeout is set here to avoid the build hanging indefinitely, still allowing
|
||||
# for retries.
|
||||
TIMEOUT_CMD=""
|
||||
if [ -n "${build_timeout_seconds}" ]; then
|
||||
TIMEOUT_CMD="timeout ${build_timeout_seconds}"
|
||||
fi
|
||||
|
||||
if is_ppc64le; then
|
||||
LOCAL_ARG="$LOCAL_ARG --jobs=48"
|
||||
fi
|
||||
|
||||
if is_s390x; then
|
||||
LOCAL_ARG="$LOCAL_ARG --jobs=16"
|
||||
fi
|
||||
# Mode-specific LOCAL_ARG and release flag
|
||||
LOCAL_ARG="$(bazel_evergreen_shutils::compute_local_arg test)"
|
||||
|
||||
# If we are doing a patch build or we are building a non-push
|
||||
# build on the waterfall, then we don't need the --release
|
||||
# flag. Otherwise, this is potentially a build that "leaves
|
||||
# the building", so we do want that flag.
|
||||
if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then
|
||||
echo "This is a non-release build."
|
||||
else
|
||||
LOCAL_ARG="$LOCAL_ARG --config=public-release"
|
||||
fi
|
||||
LOCAL_ARG="$(bazel_evergreen_shutils::maybe_release_flag "$LOCAL_ARG")"
|
||||
|
||||
if [ -n "${test_timeout_sec}" ]; then
|
||||
# s390x and ppc64le often run slower than other architectures
|
||||
if is_s390x_or_ppc64le; then
|
||||
test_timeout_sec=$(($test_timeout_sec * 4))
|
||||
fi
|
||||
bazel_args="${bazel_args} --test_timeout=${test_timeout_sec}"
|
||||
fi
|
||||
# Possibly scale test timeout and append to bazel_args
|
||||
bazel_evergreen_shutils::maybe_scale_test_timeout_and_append
|
||||
|
||||
ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} --define=MONGO_VERSION=${version} ${patch_compile_flags}"
|
||||
echo ${ALL_FLAGS} >.bazel_build_flags
|
||||
# Build the shared flags and persist the --config subset
|
||||
ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${bazel_args:-} ${bazel_compile_flags:-} ${task_compile_flags:-} --define=MONGO_VERSION=${version} ${patch_compile_flags:-}"
|
||||
echo "${ALL_FLAGS}" >.bazel_build_flags
|
||||
|
||||
# to capture exit codes
|
||||
set +o errexit
|
||||
|
||||
# Retry the build since it's deterministic and may fail due to transient issues.
|
||||
for i in {1..3}; do
|
||||
eval ${TIMEOUT_CMD} ${BAZEL_BINARY} build ${ALL_FLAGS} ${targets} && RET=0 && break || RET=$? && sleep 1
|
||||
if [ $RET -eq 124 ]; then
|
||||
echo "Bazel build timed out after ${build_timeout_seconds} seconds, retrying..."
|
||||
else
|
||||
echo "Bazel build failed, retrying..."
|
||||
fi
|
||||
$BAZEL_BINARY shutdown
|
||||
done
|
||||
# Build then test with retries.
|
||||
bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \
|
||||
build ${ALL_FLAGS} ${targets}
|
||||
RET=$?
|
||||
|
||||
for i in {1..3}; do
|
||||
eval ${TIMEOUT_CMD} ${BAZEL_BINARY} test ${ALL_FLAGS} ${targets} 2>&1 | tee bazel_stdout.log &&
|
||||
RET=0 && break || RET=$? && sleep 1
|
||||
if [ $RET -eq 124 ]; then
|
||||
echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..."
|
||||
else
|
||||
echo "Errors were found during the bazel test, failing the execution"
|
||||
break
|
||||
if [[ "$RET" == "0" ]]; then
|
||||
|
||||
bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \
|
||||
test ${ALL_FLAGS} ${targets}
|
||||
RET=$?
|
||||
|
||||
if [[ "$RET" -eq 124 ]]; then
|
||||
echo "Bazel timed out after ${build_timeout_seconds:-<unspecified>} seconds."
|
||||
elif [[ "$RET" != "0" ]]; then
|
||||
echo "Errors were found during bazel test, failing the execution"
|
||||
fi
|
||||
fi
|
||||
$BAZEL_BINARY shutdown
|
||||
done
|
||||
|
||||
set -o errexit
|
||||
|
||||
if [[ $RET != 0 ]]; then
|
||||
# The --config flag needs to stay consistent between invocations to avoid evicting the previous results.
|
||||
# Strip out anything that isn't a --config flag that could interfere with the run command.
|
||||
CONFIG_FLAGS=$(echo "${ALL_FLAGS}" | tr ' ' '\n' | grep -- '--config' | tr '\n' ' ')
|
||||
|
||||
eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:gather_failed_unittests
|
||||
if [[ "$RET" != "0" ]]; then
|
||||
CONFIG_FLAGS="$(bazel_evergreen_shutils::extract_config_flags "${ALL_FLAGS}")"
|
||||
eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:gather_failed_unittests || true
|
||||
fi
|
||||
|
||||
exit $RET
|
||||
: "${RET:=1}"
|
||||
exit "${RET}"
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
function is_macos() {
|
||||
local -r os="$(uname -s | tr '[:upper:]' '[:lower:]')"
|
||||
[[ "${os}" == "darwin" ]] && return 0 || return 1
|
||||
}
|
||||
|
||||
function is_ppc64le() {
|
||||
local -r arch="$(uname -m)"
|
||||
[[ "${arch}" == "ppc64le" || "${arch}" == "ppc64" || "${arch}" == "ppc" ]] && return 0 || return 1
|
||||
}
|
||||
|
||||
function is_s390x() {
|
||||
local -r arch="$(uname -m)"
|
||||
[[ "${arch}" == "s390x" || "${arch}" == "s390" ]] && return 0 || return 1
|
||||
}
|
||||
|
||||
function is_s390x_or_ppc64le() {
|
||||
(is_ppc64le || is_s390x) && return 0 || return 1
|
||||
}
|
||||
|
||||
function bazel_get_binary_path() {
|
||||
if is_macos; then
|
||||
echo "bazel"
|
||||
elif is_s390x_or_ppc64le ||
|
||||
grep -q "ID=debian" /etc/os-release ||
|
||||
grep -q 'ID="sles"' /etc/os-release; then
|
||||
echo "bazel/bazelisk.py"
|
||||
else
|
||||
echo "bazel"
|
||||
fi
|
||||
}
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
|
||||
. "$DIR/prelude.sh"
|
||||
|
||||
cd src
|
||||
|
||||
activate_venv
|
||||
if [ "Windows_NT" = "$OS" ]; then
|
||||
vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/"
|
||||
cd "$vcvars" && cmd /K "vcvarsall.bat amd64 && cd ${workdir}\src"
|
||||
fi
|
||||
python -m pip install ninja
|
||||
ninja install-core
|
||||
|
|
@ -12,8 +12,8 @@ cd src
|
|||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
source ./evergreen/bazel_utility_functions.sh
|
||||
BAZEL_BINARY=$(bazel_get_binary_path)
|
||||
source ./evergreen/bazel_evergreen_shutils.sh
|
||||
BAZEL_BINARY=$(bazel_evergreen_shutils::bazel_get_binary_path)
|
||||
|
||||
# Queries all resmoke_config targets: kind(resmoke_config, //...)
|
||||
# and outputs YAML key-value pair created by the starlark expression for each target.
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ cd src
|
|||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
source ./evergreen/bazel_utility_functions.sh
|
||||
BAZEL_BINARY=$(bazel_get_binary_path)
|
||||
source ./evergreen/bazel_evergreen_shutils.sh
|
||||
BAZEL_BINARY=$(bazel_evergreen_shutils::bazel_get_binary_path)
|
||||
|
||||
echo "suite_config: $(${BAZEL_BINARY} cquery ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \
|
||||
--define=MONGO_VERSION=${version} ${patch_compile_flags} ${suite}_config --output files)" >suite_config_expansion.yml
|
||||
|
|
|
|||
|
|
@ -16,9 +16,9 @@ set -o verbose
|
|||
|
||||
activate_venv
|
||||
|
||||
source ./evergreen/bazel_utility_functions.sh
|
||||
source ./evergreen/bazel_evergreen_shutils.sh
|
||||
|
||||
BAZEL_BINARY=$(bazel_get_binary_path)
|
||||
BAZEL_BINARY=$(bazel_evergreen_shutils::bazel_get_binary_path)
|
||||
|
||||
# Timeout is set here to avoid the build hanging indefinitely, still allowing
|
||||
# for retries.
|
||||
|
|
|
|||
Loading…
Reference in New Issue