mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-112016 DRY and make bazel CI scripts restart on OOM (#42300) 

GitOrigin-RevId: 6ffb7cd551d8ba59ab93fdeb7b8c4f74fcf2cea8
This commit is contained in:
Daniel Moody 2025-10-07 14:10:35 -05:00 committed by MongoDB Bot
parent fd9c122a0d
commit 612b418551
12 changed files with 465 additions and 287 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
evergreen/BUILD.bazel
View File

@ -16,11 +16,6 @@ sh_binary(
srcs = ["antithesis_image_build_and_push.sh"], srcs = ["antithesis_image_build_and_push.sh"],
) )
sh_binary(
name = "bazel_RBE_supported",
srcs = ["bazel_RBE_supported.sh"],
)
sh_binary( sh_binary(
name = "bazel_compile", name = "bazel_compile",
srcs = ["bazel_compile.sh"], srcs = ["bazel_compile.sh"],
@ -31,24 +26,19 @@ sh_binary(
srcs = ["bazel_coverage.sh"], srcs = ["bazel_coverage.sh"],
) )
sh_binary(
name = "bazel_get_binary_path",
srcs = ["bazel_get_binary_path.sh"],
)
sh_binary( sh_binary(
name = "bazel_run", name = "bazel_run",
srcs = ["bazel_run.sh"], srcs = ["bazel_run.sh"],
) )
sh_binary( sh_binary(
name = "bazel_scons_diff", name = "bazel_test",
srcs = ["bazel_scons_diff.sh"], srcs = ["bazel_test.sh"],
) )
sh_binary( sh_binary(
name = "bazel_utility_functions", name = "bazel_evergreen_shutils",
srcs = ["bazel_utility_functions.sh"], srcs = ["bazel_evergreen_shutils.sh"],
) )
sh_binary( sh_binary(
@ -76,11 +66,6 @@ sh_binary(
srcs = ["cleanup_environment.sh"], srcs = ["cleanup_environment.sh"],
) )
sh_binary(
name = "compile_ninja",
srcs = ["compile_ninja.sh"],
)
sh_binary( sh_binary(
name = "compiled_binaries_get", name = "compiled_binaries_get",
srcs = ["compiled_binaries_get.sh"], srcs = ["compiled_binaries_get.sh"],

7
evergreen/bazel.sh
View File

@ -1,7 +0,0 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
source "$DIR/bazel_utility_functions.sh"
(
cd $DIR/..
exec $(bazel_get_binary_path) "$@"
)

11
evergreen/bazel_RBE_supported.sh
View File

@ -1,11 +0,0 @@
bazel_rbe_supported() {
OS="$(uname)"
ARCH="$(uname -m)"
if [ "$ARCH" == "aarch64" ] || [ "$ARCH" == "arm64" ] || [ "$ARCH" == "x86_64" ]; then
return 0
else
return 1
fi
}

129
evergreen/bazel_compile.sh
View File

@ -11,57 +11,49 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src
set -o errexit set -o errexit
set -o verbose set -o verbose
set -o pipefail
activate_venv . "$DIR/bazel_evergreen_shutils.sh"
bazel_evergreen_shutils::activate_and_cd_src
bazel_evergreen_shutils::export_ssl_paths_if_needed
# if build_patch_id is passed, try to download binaries from specified # if build_patch_id is passed, try to download binaries from specified
# evergreen patch. # evergreen patch.
build_patch_id="${build_patch_id:-${reuse_compile_from}}" build_patch_id="${build_patch_id:-${reuse_compile_from}}"
if [ -n "${build_patch_id}" ]; then if [[ -n "${build_patch_id}" ]]; then
echo "build_patch_id detected, trying to skip task" echo "build_patch_id detected, trying to skip task"
# On windows we change the extension to zip # On windows we change the extension to zip
if [ -z "${ext}" ]; then if [[ -z "${ext:-}" ]]; then ext="tgz"; fi
ext="tgz"
fi
extra_db_contrib_args="" extra_db_contrib_args=""
# get the platform of the dist archive. This is needed if # get the platform of the dist archive. This is needed if
# db-contrib-tool cannot autodetect the platform of the ec2 instance. # db-contrib-tool cannot autodetect the platform of the ec2 instance.
regex='MONGO_DISTMOD=([a-z0-9]*)' regex='MONGO_DISTMOD=([a-z0-9]*)'
if [[ ${bazel_compile_flags} =~ ${regex} ]]; then if [[ ${bazel_compile_flags:-} =~ ${regex} ]]; then
extra_db_contrib_args="${extra_db_contrib_args} --platform=${BASH_REMATCH[1]}" extra_db_contrib_args="${extra_db_contrib_args} --platform=${BASH_REMATCH[1]}"
fi fi
download_dir="./tmp_db_contrib_tool_download_dir" download_dir="./tmp_db_contrib_tool_download_dir"
rm -rf ${download_dir} rm -rf "${download_dir}"
if [ "${task_name}" = "archive_dist_test" ]; then invocation=""
if [[ "${task_name:-}" == "archive_dist_test" ]]; then
file_name="dist-test-stripped.${ext}" file_name="dist-test-stripped.${ext}"
invocation="db-contrib-tool setup-repro-env ${build_patch_id} \ invocation="db-contrib-tool setup-repro-env ${build_patch_id} --variant=${compile_variant} --extractDownloads=False --binariesName=${file_name} --installDir=${download_dir} ${extra_db_contrib_args}"
--variant=${compile_variant} --extractDownloads=False \
--binariesName=${file_name} --installDir=${download_dir} ${extra_db_contrib_args}"
fi fi
if [[ "${task_name:-}" == "archive_dist_test_debug" ]]; then
if [ "${task_name}" = "archive_dist_test_debug" ]; then
file_name="dist-test-debug.${ext}" file_name="dist-test-debug.${ext}"
invocation="db-contrib-tool setup-repro-env ${build_patch_id} \ invocation="db-contrib-tool setup-repro-env ${build_patch_id} --variant=${compile_variant} --extractDownloads=False --debugsymbolsName=${file_name} --installDir=${download_dir} --skipBinaries --downloadSymbols ${extra_db_contrib_args}"
--variant=${compile_variant} --extractDownloads=False \
--debugsymbolsName=${file_name} --installDir=${download_dir} \
--skipBinaries --downloadSymbols ${extra_db_contrib_args}"
fi fi
if [ -n "${invocation}" ]; then if [[ -n "${invocation}" ]]; then
setup_db_contrib_tool setup_db_contrib_tool
echo "db-contrib-tool invocation: ${invocation}" echo "db-contrib-tool invocation: ${invocation}"
eval ${invocation} if ! eval ${invocation}; then
if [ $? -ne 0 ]; then
echo "Could not retrieve files with db-contrib-tool" echo "Could not retrieve files with db-contrib-tool"
exit 1 exit 1
fi fi
@ -79,10 +71,10 @@ fi
# --build-mongot is a compile flag used by the evergreen build variants that run end-to-end search # --build-mongot is a compile flag used by the evergreen build variants that run end-to-end search
# suites, as it downloads the necessary mongot binary. # suites, as it downloads the necessary mongot binary.
if [ "${build_mongot}" = "true" ]; then if [[ "${build_mongot:-}" == "true" ]]; then
setup_db_contrib_tool setup_db_contrib_tool
use_db_contrib_tool_mongot use_db_contrib_tool_mongot
bazel_args="${bazel_args} --include_mongot=True" bazel_args="${bazel_args:-} --include_mongot=True"
fi fi
# This is hacky way to pass off build time from archive_dist_test to archive_dist_test_debug # This is hacky way to pass off build time from archive_dist_test to archive_dist_test_debug
@ -90,12 +82,12 @@ fi
# We then create the symbols normally in archive_dist_test_debug. We have to force the # We then create the symbols normally in archive_dist_test_debug. We have to force the
# build-id for debugging as they will be different when -Wl,-S is passed in. # build-id for debugging as they will be different when -Wl,-S is passed in.
# The relinked binaries should still be hash identical when stripped with strip # The relinked binaries should still be hash identical when stripped with strip
if [ "${skip_debug_link}" = "true" ]; then if [[ "${skip_debug_link:-}" == "true" ]]; then
export compile_variant="${compile_variant}" export compile_variant="${compile_variant}"
export version_id="${version_id}" export version_id="${version_id}"
if [ "${task_name}" = "archive_dist_test" ]; then if [[ "${task_name:-}" == "archive_dist_test" ]]; then
task_compile_flags="${task_compile_flags} --simple_build_id=True --features=strip_debug --separate_debug=False" task_compile_flags="${task_compile_flags:-} --simple_build_id=True --features=strip_debug --separate_debug=False"
if [ "${remote_link}" = "true" ]; then if [[ "${remote_link:-}" == "true" ]]; then
ARCH=$(uname -m) ARCH=$(uname -m)
# Remote linking is currently only supported on arm # Remote linking is currently only supported on arm
if [[ "$ARCH" == "arm64" || "$ARCH" == "aarch64" ]]; then if [[ "$ARCH" == "arm64" || "$ARCH" == "aarch64" ]]; then
@ -103,66 +95,45 @@ if [ "${skip_debug_link}" = "true" ]; then
fi fi
fi fi
fi fi
if [ "${task_name}" = "archive_dist_test_debug" ]; then if [[ "${task_name:-}" == "archive_dist_test_debug" ]]; then
task_compile_flags="${task_compile_flags} --simple_build_id=True" task_compile_flags="${task_compile_flags:-} --simple_build_id=True"
fi fi
fi fi
set -o pipefail
# Use `eval` to force evaluation of the environment variables in the echo statement: # Use `eval` to force evaluation of the environment variables in the echo statement:
eval echo "Execution environment: Targets: ${targets}" eval echo "Execution environment: Targets: ${targets}"
source ./evergreen/bazel_utility_functions.sh BAZEL_BINARY="$(bazel_evergreen_shutils::bazel_get_binary_path)"
source ./evergreen/bazel_RBE_supported.sh
if [[ "${evergreen_remote_exec}" != "on" ]]; then # Compose LOCAL_ARG and release flags
LOCAL_ARG="$LOCAL_ARG --jobs=auto" LOCAL_ARG="$(bazel_evergreen_shutils::compute_local_arg build)"
fi
BAZEL_BINARY=$(bazel_get_binary_path)
# Timeout is set here to avoid the build hanging indefinitely, still allowing
# for retries.
TIMEOUT_CMD=""
if [ -n "${build_timeout_seconds}" ]; then
TIMEOUT_CMD="timeout ${build_timeout_seconds}"
elif [[ "${evergreen_remote_exec}" == "on" ]]; then
# Timeout remote execution runs in 60 minutes as a workaround for
# scheduling timeout bugs
TIMEOUT_CMD="timeout 3600"
fi
if is_ppc64le; then
LOCAL_ARG="$LOCAL_ARG --jobs=48"
fi
if is_s390x; then
LOCAL_ARG="$LOCAL_ARG --jobs=16"
fi
# If we are doing a patch build or we are building a non-push # If we are doing a patch build or we are building a non-push
# build on the waterfall, then we don't need the --release # build on the waterfall, then we don't need the --release
# flag. Otherwise, this is potentially a build that "leaves # flag. Otherwise, this is potentially a build that "leaves
# the building", so we do want that flag. # the building", so we do want that flag.
if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then LOCAL_ARG="$(bazel_evergreen_shutils::maybe_release_flag "$LOCAL_ARG")"
echo "This is a non-release build."
else # Ensure server is up and print PID
LOCAL_ARG="$LOCAL_ARG --config=public-release" bazel_evergreen_shutils::ensure_server_and_print_pid "$BAZEL_BINARY"
# Build flags line
ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${bazel_args:-} ${bazel_compile_flags:-} ${task_compile_flags:-} --define=MONGO_VERSION=${version} ${patch_compile_flags:-}"
echo "${ALL_FLAGS}" >.bazel_build_flags
set +o errexit
bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \
build ${ALL_FLAGS} ${targets}
RET=$?
set -o errexit
if [[ "$RET" -eq 124 ]]; then
echo "Bazel build timed out after ${build_timeout_seconds:-<unspecified>} seconds."
elif [[ "$RET" != "0" ]]; then
echo "Bazel build failed after retries."
fi fi
for i in {1..3}; do : "${RET:=1}"
eval ${TIMEOUT_CMD} $BAZEL_BINARY build --verbose_failures $LOCAL_ARG ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \ exit "${RET}"
--define=MONGO_VERSION=${version} ${patch_compile_flags} ${targets} 2>&1 | tee bazel_stdout.log &&
RET=0 && break || RET=$? && sleep 60
if [ $RET -eq 124 ]; then
echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..."
else
echo "Errors were found during the bazel run, here are the errors:" 1>&2
grep "ERROR:" bazel_stdout.log 1>&2
echo "Bazel failed to execute, retrying..."
fi
$BAZEL_BINARY shutdown
done
exit $RET

335
evergreen/bazel_evergreen_shutils.sh Normal file
View File

@ -0,0 +1,335 @@
# Common helpers for CI Bazel scripts (build/run/test)
set -o errexit
set -o pipefail
# --- Pre-flight (assumes prelude.sh already sourced by caller) -------------
bazel_evergreen_shutils::activate_and_cd_src() {
cd src
set -o verbose
activate_venv
}
# --- Distro quirks -----------------------------------------------------
bazel_evergreen_shutils::export_ssl_paths_if_needed() {
if [[ -f /etc/os-release ]]; then
local DISTRO
DISTRO=$(awk -F '[="]*' '/^PRETTY_NAME/ { print $2 }' </etc/os-release)
if [[ "$DISTRO" == "Amazon Linux 2" ]]; then
export SSL_CERT_DIR=/etc/pki/tls/certs
export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
elif [[ "$DISTRO" == "Red Hat Enterprise Linux"* ]]; then
export SSL_CERT_DIR=/etc/pki/ca-trust/extracted/pem
export SSL_CERT_FILE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
fi
fi
}
bazel_evergreen_shutils::is_macos() {
local -r os="$(uname -s | tr '[:upper:]' '[:lower:]')"
[[ "${os}" == "darwin" ]] && return 0 || return 1
}
bazel_evergreen_shutils::is_ppc64le() {
local -r arch="$(uname -m)"
[[ "${arch}" == "ppc64le" || "${arch}" == "ppc64" || "${arch}" == "ppc" ]] && return 0 || return 1
}
bazel_evergreen_shutils::is_s390x() {
local -r arch="$(uname -m)"
[[ "${arch}" == "s390x" || "${arch}" == "s390" ]] && return 0 || return 1
}
bazel_evergreen_shutils::is_s390x_or_ppc64le() {
if bazel_evergreen_shutils::is_ppc64le || bazel_evergreen_shutils::is_s390x; then
return 0
else
return 1
fi
}
bazel_evergreen_shutils::bazel_get_binary_path() {
if bazel_evergreen_shutils::is_macos; then
echo "bazel"
elif bazel_evergreen_shutils::is_s390x_or_ppc64le ||
grep -q "ID=debian" /etc/os-release ||
grep -q 'ID="sles"' /etc/os-release; then
echo "bazel/bazelisk.py"
else
echo "bazel"
fi
}
# --- RBE/local flags + arch tuning ----------------------------------------
bazel_evergreen_shutils::bazel_rbe_supported() {
local OS ARCH
OS="$(uname)"
ARCH="$(uname -m)"
if [ "$ARCH" == "aarch64" ] || [ "$ARCH" == "arm64" ] || [ "$ARCH" == "x86_64" ]; then
return 0
else
return 1
fi
}
# Requires: evergreen_remote_exec, task_name (for tests), bazel_args vars optionally.
bazel_evergreen_shutils::compute_local_arg() {
local mode="${1:-build}" # build|test|run
local local_arg=""
if [[ "${evergreen_remote_exec:-}" != "on" ]]; then
local_arg+=" --jobs=auto"
elif [[ "$mode" == "test" && "${task_name:-}" == "unit_tests" ]]; then
local_arg+=" --config=remote_test"
fi
if bazel_evergreen_shutils::is_ppc64le; then
local_arg+=" --jobs=48"
fi
if bazel_evergreen_shutils::is_s390x; then
local_arg+=" --jobs=16"
fi
# For run-mode, if RBE isn't supported or is disabled explicitly, force local config.
if [[ "$mode" == "run" ]]; then
if ! bazel_evergreen_shutils::bazel_rbe_supported || [[ "${evergreen_remote_exec:-}" != "on" ]]; then
# Keep compatibility with existing pattern:
local_arg+=" --config=local"
fi
fi
echo "$local_arg"
}
# Keeps only --config flags from a flag string (used to persist consistency)
bazel_evergreen_shutils::extract_config_flags() {
local all="$*"
awk '{
for (i=1;i<=NF;i++) if ($i ~ /^--config(=|$)/) printf "%s ", $i
}' <<<"$all"
}
# Adds --config=public-release if this is a release-ish build.
bazel_evergreen_shutils::maybe_release_flag() {
local local_arg="$1"
if [[ "${is_patch:-}" == "true" || -z "${push_bucket:-}" || "${compiling_for_test:-}" == "true" ]]; then
echo "$local_arg" # non-release
else
echo "$local_arg --config=public-release"
fi
}
# --- Timeouts --------------------------------------------------------------
# Timeout helper: returns a "timeout <secs>" prefix or empty string.
# Prints a one-time warning to stderr if a timeout was requested but no
# timeout binary is available. Supports macOS 'gtimeout' if installed.
bazel_evergreen_shutils::timeout_prefix() {
local fallback_remote="${1:-}" # "on" = use 3600s default for remote builds
local need_timeout="" # "explicit" | "fallback" | ""
local timeout_bin=""
# Do we want a timeout?
if [[ -n "${build_timeout_seconds:-}" ]]; then
need_timeout="explicit"
elif [[ "$fallback_remote" == "on" ]]; then
need_timeout="fallback"
fi
# Find a timeout binary (GNU coreutils 'timeout' or macOS 'gtimeout')
if command -v timeout >/dev/null 2>&1; then
timeout_bin="timeout"
elif command -v gtimeout >/dev/null 2>&1; then
timeout_bin="gtimeout"
fi
# If needed but unavailable, warn once and return empty
if [[ -n "$need_timeout" && -z "$timeout_bin" ]]; then
if [[ -z "${_BAZEL_EVG_TIMEOUT_WARNED:-}" ]]; then
if [[ "$need_timeout" == "explicit" ]]; then
echo "[warn] 'timeout' not found; requested ${build_timeout_seconds}s timeout will be ignored." >&2
else
echo "[warn] 'timeout' not found; remote-build fallback timeout (3600s) will be ignored." >&2
fi
# Helpful hint for macOS users
if bazel_evergreen_shutils::is_macos; then
echo "[hint] On macOS, install coreutils: 'brew install coreutils' (provides 'gtimeout')." >&2
fi
_BAZEL_EVG_TIMEOUT_WARNED=1
fi
echo "" # no timeout prefix
return 0
fi
# Produce the prefix if we have a binary
if [[ -n "$timeout_bin" ]]; then
if [[ "$need_timeout" == "explicit" ]]; then
echo "$timeout_bin ${build_timeout_seconds}"
elif [[ "$need_timeout" == "fallback" ]]; then
echo "$timeout_bin 3600"
else
echo ""
fi
else
echo ""
fi
}
# --- Bazel server lifecycle & OOM-detect retry -----------------------------
bazel_evergreen_shutils::bazel_output_base() {
local BAZEL_BINARY="$1"
"$BAZEL_BINARY" info output_base 2>/dev/null
}
bazel_evergreen_shutils::bazel_pidfile_path() {
local BAZEL_BINARY="$1"
local ob
ob="$(bazel_evergreen_shutils::bazel_output_base "$BAZEL_BINARY")" || return 1
echo "${ob}/server/server.pid.txt"
}
bazel_evergreen_shutils::is_bazel_server_running() {
local BAZEL_BINARY="$1"
local pf pid
pf="$(bazel_evergreen_shutils::bazel_pidfile_path "$BAZEL_BINARY")" || return 1
[[ -f "$pf" ]] || return 1
pid="$(cat "$pf" 2>/dev/null || true)"
[[ -n "$pid" ]] || return 1
if kill -0 "$pid" 2>/dev/null; then
return 0
else
return 1
fi
}
bazel_evergreen_shutils::print_bazel_server_pid() {
local BAZEL_BINARY="$1"
local pf pid
pf="$(bazel_evergreen_shutils::bazel_pidfile_path "$BAZEL_BINARY")" || {
echo "Bazel server pidfile not found (output_base: $(bazel_evergreen_shutils::bazel_output_base "$BAZEL_BINARY" 2>/dev/null || true))"
return 0
}
if [[ -f "$pf" ]]; then
pid="$(cat "$pf" 2>/dev/null || true)"
echo "Bazel server pidfile: $pf (PID=${pid:-unknown})"
else
echo "Bazel server pidfile not found yet (output_base: $(bazel_evergreen_shutils::bazel_output_base "$BAZEL_BINARY" 2>/dev/null || true))"
fi
}
# Starts server (if needed) and prints PID. Safe to call multiple times.
bazel_evergreen_shutils::ensure_server_and_print_pid() {
local BAZEL_BINARY="$1"
_IGN=$("$BAZEL_BINARY" info >/dev/null 2>&1 || true)
bazel_evergreen_shutils::print_bazel_server_pid "$BAZEL_BINARY"
}
# Generic retry wrapper:
# $1: attempts
# $3: bazel binary
# $4..: full bazel subcommand + args (e.g. "build --verbose_failures ...")
# Special handling:
# - exit 124 -> timeout
# - server death (pid missing) -> restart, then retry
# Returns with global RET set.
bazel_evergreen_shutils::retry_bazel_cmd() {
local attempts="$1"
shift
local BAZEL_BINARY="$1"
shift
local timeout_str="$(bazel_evergreen_shutils::timeout_prefix "${evergreen_remote_exec:-}")"
# Everything else is the Bazel subcommand + flags (and possibly redirections/pipes).
# We *intentionally* keep it as raw words and reassemble to a single string for eval.
local raw_rest=("$@")
# Once we detect an OOM/server-death, we enable the guard for subsequent attempts.
local use_oom_guard=false
local -r OOM_GUARD_FLAG='--local_resources=HOST_CPUS*.5'
# Helper: does the current command string already include a local_resources flag?
_cmd_has_local_resources() {
[[ "$1" == *"--local_resources"* ]]
}
local RET=1
for i in $(seq 1 "$attempts"); do
echo "Attempt ${i}/${attempts}…" >&2
# Ensure/refresh server & pid before we run (helps produce a fresh pidfile too).
if ! bazel_evergreen_shutils::is_bazel_server_running "$BAZEL_BINARY"; then
echo "[retry ${i}] Bazel server not running (likely OOM/killed); restarting…" >&2
"$BAZEL_BINARY" info >/dev/null 2>&1 || true
bazel_evergreen_shutils::print_bazel_server_pid "$BAZEL_BINARY" >&2
fi
# Reassemble the caller’s words into a single command string for eval.
# We deliberately do *not* try to be clever here—this restores legacy behavior
# where quoted pieces inside variables (e.g., --base_dir="..") are honored by the shell.
local cmd="\"$BAZEL_BINARY\""
local w
for w in "${raw_rest[@]}"; do
cmd+=" $w"
done
# If OOM guard is enabled and not already present, append it.
# (Safe with eval; if the caller added redirections earlier, this is still just an arg.)
if $use_oom_guard && ! _cmd_has_local_resources "$cmd"; then
echo "[retry ${i}] Applying OOM guard: ${OOM_GUARD_FLAG}" >&2
cmd+=" ${OOM_GUARD_FLAG}"
fi
# Prefix timeout, if any.
if [[ -n "$timeout_str" ]]; then
cmd="${timeout_str} ${cmd}"
fi
# Run it.
# NOTE: We *do not* add any redirections here; caller controls logging completely.
if eval "$cmd"; then
RET=0
break
else
RET=$?
fi
# Classify failure & decide on guard for next attempt.
[[ $RET -eq 124 ]] && echo "Bazel timed out." >&2
if ! bazel_evergreen_shutils::is_bazel_server_running "$BAZEL_BINARY"; then
echo "[retry ${i}] Bazel server down (OOM/killed). Enabling OOM guard for next attempt and restarting…" >&2
use_oom_guard=true
"$BAZEL_BINARY" shutdown || true
"$BAZEL_BINARY" info >/dev/null 2>&1 || true
bazel_evergreen_shutils::print_bazel_server_pid "$BAZEL_BINARY" >&2
else
echo "Bazel failed (exit=$RET); restarting server before retry…" >&2
"$BAZEL_BINARY" shutdown || true
fi
sleep 60
done
return "$RET"
}
# --- Test helpers ----------------------------------------------------------
# Multiplies test timeout on slow arches and appends to bazel_args
bazel_evergreen_shutils::maybe_scale_test_timeout_and_append() {
if [[ -n "${test_timeout_sec:-}" ]]; then
local scaled="$test_timeout_sec"
if bazel_evergreen_shutils::is_s390x_or_ppc64le; then
scaled=$((test_timeout_sec * 4))
fi
bazel_args="${bazel_args:-} --test_timeout=${scaled}"
fi
}

85
evergreen/bazel_run.sh
View File

@ -7,73 +7,54 @@
# * ${env} - Env variable string to set (ex. ENV_VAR_ABC=123) # * ${env} - Env variable string to set (ex. ENV_VAR_ABC=123)
# * ${redact_args} - If set, redact the args in the report # * ${redact_args} - If set, redact the args in the report
# Needed for evergreen scripts that use evergreen expansions and utility methods.
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src
set -o errexit set -o errexit
set -o verbose set -o verbose
set -o pipefail
activate_venv . "$DIR/bazel_evergreen_shutils.sh"
bazel_evergreen_shutils::activate_and_cd_src
bazel_evergreen_shutils::export_ssl_paths_if_needed
# Use `eval` to force evaluation of the environment variables in the echo statement: # Use `eval` to force evaluation of the environment variables in the echo statement:
eval echo "Execution environment: Args: ${args} Target: ${target} Env: ${env} redact_args: ${redact_args}" eval echo "Execution environment: Args: ${args} Target: ${target} Env: ${env} redact_args: ${redact_args}"
source ./evergreen/bazel_utility_functions.sh BAZEL_BINARY="$(bazel_evergreen_shutils::bazel_get_binary_path)"
source ./evergreen/bazel_RBE_supported.sh
if bazel_rbe_supported; then # Build LOCAL_ARG for run-mode
LOCAL_ARG="" LOCAL_ARG="$(bazel_evergreen_shutils::compute_local_arg run)"
else # Honor .bazel_build_flags --config lines (do not evict previous cache)
LOCAL_ARG="--config=local"
fi
if [[ "${evergreen_remote_exec}" != "on" ]]; then
LOCAL_ARG="--config=local"
fi
BAZEL_BINARY=$(bazel_get_binary_path)
# AL2 stores certs in a nonstandard location
if [[ -f /etc/os-release ]]; then
DISTRO=$(awk -F '[="]*' '/^PRETTY_NAME/ { print $2 }' </etc/os-release)
if [[ $DISTRO == "Amazon Linux 2" ]]; then
export SSL_CERT_DIR=/etc/pki/tls/certs
export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
elif [[ $DISTRO == "Red Hat Enterprise Linux"* ]]; then
export SSL_CERT_DIR=/etc/pki/ca-trust/extracted/pem
export SSL_CERT_FILE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
fi
fi
if [[ -n "$redact_args" ]]; then
INVOCATION_WITH_REDACTION="${target}"
else
INVOCATION_WITH_REDACTION="${target} ${args}"
fi
# The --config flag needs to stay consistent between invocations to avoid evicting the previous results.
# Strip out anything that isn't a --config flag that could interfere with the run command.
ALL_FLAGS="" ALL_FLAGS=""
if [[ -f .bazel_build_flags ]]; then if [[ -f .bazel_build_flags ]]; then
ALL_FLAGS=$(<.bazel_build_flags) ALL_FLAGS="$(<.bazel_build_flags)"
fi fi
CONFIG_FLAGS=$(echo "${ALL_FLAGS}" | tr ' ' '\n' | grep -- '--config' | tr '\n' ' ') CONFIG_FLAGS="$(bazel_evergreen_shutils::extract_config_flags "${ALL_FLAGS}")"
LOCAL_ARG="${CONFIG_FLAGS} ${LOCAL_ARG}" LOCAL_ARG="${CONFIG_FLAGS} ${LOCAL_ARG}"
# Print command being run to file that can be uploaded INVOCATION_WITH_REDACTION="${target}"
echo "bazel run --verbose_failures ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >>bazel-invocation.txt if [[ -z "${redact_args:-}" ]]; then
INVOCATION_WITH_REDACTION+=" ${args}"
fi
# Run bazel command, retrying up to five times # Record invocation
MAX_ATTEMPTS=5 echo "bazel run --verbose_failures ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >bazel-invocation.txt
for ((i = 1; i <= $MAX_ATTEMPTS; i++)); do
eval $env $BAZEL_BINARY run --verbose_failures ${LOCAL_ARG} ${target} ${args} >>bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 10
if [ $i -lt $MAX_ATTEMPTS ]; then echo "Bazel failed to execute, retrying ($(($i + 1)) of $MAX_ATTEMPTS attempts)... " >>bazel_output.log 2>&1; fi
$BAZEL_BINARY shutdown
done
$python ./buildscripts/simple_report.py --test-name "bazel run ${INVOCATION_WITH_REDACTION}" --log-file bazel_output.log --exit-code $RET # capture exit code
exit $RET set +o errexit
bazel_evergreen_shutils::retry_bazel_cmd 5 "$BAZEL_BINARY" \
run --verbose_failures ${LOCAL_ARG} ${target} ${args} 2>&1 | tee -a bazel_output.log
RET=${PIPESTATUS[0]}
: "${RET:=1}"
set -o errexit
# Report
$python ./buildscripts/simple_report.py \
--test-name "bazel run ${INVOCATION_WITH_REDACTION}" \
--log-file bazel_output.log \
--exit-code "${RET}"
exit "${RET}"

108
evergreen/bazel_test.sh
View File

@ -8,99 +8,65 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src
set -o errexit set -o errexit
set -o verbose set -o verbose
activate_venv
set -o pipefail set -o pipefail
. "$DIR/bazel_evergreen_shutils.sh"
bazel_evergreen_shutils::activate_and_cd_src
bazel_evergreen_shutils::export_ssl_paths_if_needed
# Use `eval` to force evaluation of the environment variables in the echo statement: # Use `eval` to force evaluation of the environment variables in the echo statement:
eval echo "Execution environment: Targets: ${targets}" eval echo "Execution environment: Targets: ${targets}"
source ./evergreen/bazel_utility_functions.sh BAZEL_BINARY="$(bazel_evergreen_shutils::bazel_get_binary_path)"
source ./evergreen/bazel_RBE_supported.sh
LOCAL_ARG="" # Mode-specific LOCAL_ARG and release flag
if [[ "${evergreen_remote_exec}" != "on" ]]; then LOCAL_ARG="$(bazel_evergreen_shutils::compute_local_arg test)"
LOCAL_ARG="$LOCAL_ARG --jobs=auto"
elif [[ "${task_name}" == "unit_tests" ]]; then
LOCAL_ARG="$LOCAL_ARG --config=remote_test"
fi
BAZEL_BINARY=$(bazel_get_binary_path)
# Timeout is set here to avoid the build hanging indefinitely, still allowing
# for retries.
TIMEOUT_CMD=""
if [ -n "${build_timeout_seconds}" ]; then
TIMEOUT_CMD="timeout ${build_timeout_seconds}"
fi
if is_ppc64le; then
LOCAL_ARG="$LOCAL_ARG --jobs=48"
fi
if is_s390x; then
LOCAL_ARG="$LOCAL_ARG --jobs=16"
fi
# If we are doing a patch build or we are building a non-push # If we are doing a patch build or we are building a non-push
# build on the waterfall, then we don't need the --release # build on the waterfall, then we don't need the --release
# flag. Otherwise, this is potentially a build that "leaves # flag. Otherwise, this is potentially a build that "leaves
# the building", so we do want that flag. # the building", so we do want that flag.
if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then LOCAL_ARG="$(bazel_evergreen_shutils::maybe_release_flag "$LOCAL_ARG")"
echo "This is a non-release build."
else
LOCAL_ARG="$LOCAL_ARG --config=public-release"
fi
if [ -n "${test_timeout_sec}" ]; then # Possibly scale test timeout and append to bazel_args
# s390x and ppc64le often run slower than other architectures bazel_evergreen_shutils::maybe_scale_test_timeout_and_append
if is_s390x_or_ppc64le; then
test_timeout_sec=$(($test_timeout_sec * 4))
fi
bazel_args="${bazel_args} --test_timeout=${test_timeout_sec}"
fi
ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} --define=MONGO_VERSION=${version} ${patch_compile_flags}" # Build the shared flags and persist the --config subset
echo ${ALL_FLAGS} >.bazel_build_flags ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${bazel_args:-} ${bazel_compile_flags:-} ${task_compile_flags:-} --define=MONGO_VERSION=${version} ${patch_compile_flags:-}"
echo "${ALL_FLAGS}" >.bazel_build_flags
# to capture exit codes
set +o errexit set +o errexit
# Retry the build since it's deterministic and may fail due to transient issues. # Build then test with retries.
for i in {1..3}; do bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \
eval ${TIMEOUT_CMD} ${BAZEL_BINARY} build ${ALL_FLAGS} ${targets} && RET=0 && break || RET=$? && sleep 1 build ${ALL_FLAGS} ${targets}
if [ $RET -eq 124 ]; then RET=$?
echo "Bazel build timed out after ${build_timeout_seconds} seconds, retrying..."
else
echo "Bazel build failed, retrying..."
fi
$BAZEL_BINARY shutdown
done
for i in {1..3}; do if [[ "$RET" == "0" ]]; then
eval ${TIMEOUT_CMD} ${BAZEL_BINARY} test ${ALL_FLAGS} ${targets} 2>&1 | tee bazel_stdout.log &&
RET=0 && break || RET=$? && sleep 1 bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \
if [ $RET -eq 124 ]; then test ${ALL_FLAGS} ${targets}
echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..." RET=$?
else
echo "Errors were found during the bazel test, failing the execution" if [[ "$RET" -eq 124 ]]; then
break echo "Bazel timed out after ${build_timeout_seconds:-<unspecified>} seconds."
elif [[ "$RET" != "0" ]]; then
echo "Errors were found during bazel test, failing the execution"
fi fi
$BAZEL_BINARY shutdown fi
done
set -o errexit set -o errexit
if [[ $RET != 0 ]]; then # The --config flag needs to stay consistent between invocations to avoid evicting the previous results.
# The --config flag needs to stay consistent between invocations to avoid evicting the previous results. # Strip out anything that isn't a --config flag that could interfere with the run command.
# Strip out anything that isn't a --config flag that could interfere with the run command. if [[ "$RET" != "0" ]]; then
CONFIG_FLAGS=$(echo "${ALL_FLAGS}" | tr ' ' '\n' | grep -- '--config' | tr '\n' ' ') CONFIG_FLAGS="$(bazel_evergreen_shutils::extract_config_flags "${ALL_FLAGS}")"
eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:gather_failed_unittests || true
eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:gather_failed_unittests
fi fi
exit $RET : "${RET:=1}"
exit "${RET}"

30
evergreen/bazel_utility_functions.sh
View File

@ -1,30 +0,0 @@
function is_macos() {
local -r os="$(uname -s | tr '[:upper:]' '[:lower:]')"
[[ "${os}" == "darwin" ]] && return 0 || return 1
}
function is_ppc64le() {
local -r arch="$(uname -m)"
[[ "${arch}" == "ppc64le" || "${arch}" == "ppc64" || "${arch}" == "ppc" ]] && return 0 || return 1
}
function is_s390x() {
local -r arch="$(uname -m)"
[[ "${arch}" == "s390x" || "${arch}" == "s390" ]] && return 0 || return 1
}
function is_s390x_or_ppc64le() {
(is_ppc64le || is_s390x) && return 0 || return 1
}
function bazel_get_binary_path() {
if is_macos; then
echo "bazel"
elif is_s390x_or_ppc64le ||
grep -q "ID=debian" /etc/os-release ||
grep -q 'ID="sles"' /etc/os-release; then
echo "bazel/bazelisk.py"
else
echo "bazel"
fi
}

12
evergreen/compile_ninja.sh
View File

@ -1,12 +0,0 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
activate_venv
if [ "Windows_NT" = "$OS" ]; then
vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/"
cd "$vcvars" && cmd /K "vcvarsall.bat amd64 && cd ${workdir}\src"
fi
python -m pip install ninja
ninja install-core

4
evergreen/get_all_resmoke_suite_configs.sh
View File

@ -12,8 +12,8 @@ cd src
set -o errexit set -o errexit
set -o verbose set -o verbose
source ./evergreen/bazel_utility_functions.sh source ./evergreen/bazel_evergreen_shutils.sh
BAZEL_BINARY=$(bazel_get_binary_path) BAZEL_BINARY=$(bazel_evergreen_shutils::bazel_get_binary_path)
# Queries all resmoke_config targets: kind(resmoke_config, //...) # Queries all resmoke_config targets: kind(resmoke_config, //...)
# and outputs YAML key-value pair created by the starlark expression for each target. # and outputs YAML key-value pair created by the starlark expression for each target.

4
evergreen/get_resmoke_suite_config.sh
View File

@ -14,8 +14,8 @@ cd src
set -o errexit set -o errexit
set -o verbose set -o verbose
source ./evergreen/bazel_utility_functions.sh source ./evergreen/bazel_evergreen_shutils.sh
BAZEL_BINARY=$(bazel_get_binary_path) BAZEL_BINARY=$(bazel_evergreen_shutils::bazel_get_binary_path)
echo "suite_config: $(${BAZEL_BINARY} cquery ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \ echo "suite_config: $(${BAZEL_BINARY} cquery ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \
--define=MONGO_VERSION=${version} ${patch_compile_flags} ${suite}_config --output files)" >suite_config_expansion.yml --define=MONGO_VERSION=${version} ${patch_compile_flags} ${suite}_config --output files)" >suite_config_expansion.yml

4
evergreen/resmoke_tests_execute_bazel.sh
View File

@ -16,9 +16,9 @@ set -o verbose
activate_venv activate_venv
source ./evergreen/bazel_utility_functions.sh source ./evergreen/bazel_evergreen_shutils.sh
BAZEL_BINARY=$(bazel_get_binary_path) BAZEL_BINARY=$(bazel_evergreen_shutils::bazel_get_binary_path)
# Timeout is set here to avoid the build hanging indefinitely, still allowing # Timeout is set here to avoid the build hanging indefinitely, still allowing
# for retries. # for retries.