diff --git a/evergreen/BUILD.bazel b/evergreen/BUILD.bazel index 18fc751e289..575540d3771 100644 --- a/evergreen/BUILD.bazel +++ b/evergreen/BUILD.bazel @@ -16,11 +16,6 @@ sh_binary( srcs = ["antithesis_image_build_and_push.sh"], ) -sh_binary( - name = "bazel_RBE_supported", - srcs = ["bazel_RBE_supported.sh"], -) - sh_binary( name = "bazel_compile", srcs = ["bazel_compile.sh"], @@ -31,24 +26,19 @@ sh_binary( srcs = ["bazel_coverage.sh"], ) -sh_binary( - name = "bazel_get_binary_path", - srcs = ["bazel_get_binary_path.sh"], -) - sh_binary( name = "bazel_run", srcs = ["bazel_run.sh"], ) sh_binary( - name = "bazel_scons_diff", - srcs = ["bazel_scons_diff.sh"], + name = "bazel_test", + srcs = ["bazel_test.sh"], ) sh_binary( - name = "bazel_utility_functions", - srcs = ["bazel_utility_functions.sh"], + name = "bazel_evergreen_shutils", + srcs = ["bazel_evergreen_shutils.sh"], ) sh_binary( @@ -76,11 +66,6 @@ sh_binary( srcs = ["cleanup_environment.sh"], ) -sh_binary( - name = "compile_ninja", - srcs = ["compile_ninja.sh"], -) - sh_binary( name = "compiled_binaries_get", srcs = ["compiled_binaries_get.sh"], diff --git a/evergreen/bazel.sh b/evergreen/bazel.sh deleted file mode 100755 index c9761f726b9..00000000000 --- a/evergreen/bazel.sh +++ /dev/null @@ -1,7 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" - -source "$DIR/bazel_utility_functions.sh" -( - cd $DIR/.. - exec $(bazel_get_binary_path) "$@" -) diff --git a/evergreen/bazel_RBE_supported.sh b/evergreen/bazel_RBE_supported.sh deleted file mode 100644 index 54fe9d9512a..00000000000 --- a/evergreen/bazel_RBE_supported.sh +++ /dev/null @@ -1,11 +0,0 @@ -bazel_rbe_supported() { - - OS="$(uname)" - ARCH="$(uname -m)" - - if [ "$ARCH" == "aarch64" ] || [ "$ARCH" == "arm64" ] || [ "$ARCH" == "x86_64" ]; then - return 0 - else - return 1 - fi -} diff --git a/evergreen/bazel_compile.sh b/evergreen/bazel_compile.sh index e474c577123..924e437b566 100755 --- a/evergreen/bazel_compile.sh +++ b/evergreen/bazel_compile.sh @@ -11,57 +11,49 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" -cd src - set -o errexit set -o verbose +set -o pipefail -activate_venv +. "$DIR/bazel_evergreen_shutils.sh" + +bazel_evergreen_shutils::activate_and_cd_src +bazel_evergreen_shutils::export_ssl_paths_if_needed # if build_patch_id is passed, try to download binaries from specified # evergreen patch. build_patch_id="${build_patch_id:-${reuse_compile_from}}" -if [ -n "${build_patch_id}" ]; then +if [[ -n "${build_patch_id}" ]]; then echo "build_patch_id detected, trying to skip task" # On windows we change the extension to zip - if [ -z "${ext}" ]; then - ext="tgz" - fi - + if [[ -z "${ext:-}" ]]; then ext="tgz"; fi extra_db_contrib_args="" # get the platform of the dist archive. This is needed if # db-contrib-tool cannot autodetect the platform of the ec2 instance. regex='MONGO_DISTMOD=([a-z0-9]*)' - if [[ ${bazel_compile_flags} =~ ${regex} ]]; then + if [[ ${bazel_compile_flags:-} =~ ${regex} ]]; then extra_db_contrib_args="${extra_db_contrib_args} --platform=${BASH_REMATCH[1]}" fi download_dir="./tmp_db_contrib_tool_download_dir" - rm -rf ${download_dir} + rm -rf "${download_dir}" - if [ "${task_name}" = "archive_dist_test" ]; then + invocation="" + if [[ "${task_name:-}" == "archive_dist_test" ]]; then file_name="dist-test-stripped.${ext}" - invocation="db-contrib-tool setup-repro-env ${build_patch_id} \ - --variant=${compile_variant} --extractDownloads=False \ - --binariesName=${file_name} --installDir=${download_dir} ${extra_db_contrib_args}" + invocation="db-contrib-tool setup-repro-env ${build_patch_id} --variant=${compile_variant} --extractDownloads=False --binariesName=${file_name} --installDir=${download_dir} ${extra_db_contrib_args}" fi - - if [ "${task_name}" = "archive_dist_test_debug" ]; then + if [[ "${task_name:-}" == "archive_dist_test_debug" ]]; then file_name="dist-test-debug.${ext}" - invocation="db-contrib-tool setup-repro-env ${build_patch_id} \ - --variant=${compile_variant} --extractDownloads=False \ - --debugsymbolsName=${file_name} --installDir=${download_dir} \ - --skipBinaries --downloadSymbols ${extra_db_contrib_args}" + invocation="db-contrib-tool setup-repro-env ${build_patch_id} --variant=${compile_variant} --extractDownloads=False --debugsymbolsName=${file_name} --installDir=${download_dir} --skipBinaries --downloadSymbols ${extra_db_contrib_args}" fi - if [ -n "${invocation}" ]; then + if [[ -n "${invocation}" ]]; then setup_db_contrib_tool - echo "db-contrib-tool invocation: ${invocation}" - eval ${invocation} - if [ $? -ne 0 ]; then + if ! eval ${invocation}; then echo "Could not retrieve files with db-contrib-tool" exit 1 fi @@ -79,10 +71,10 @@ fi # --build-mongot is a compile flag used by the evergreen build variants that run end-to-end search # suites, as it downloads the necessary mongot binary. -if [ "${build_mongot}" = "true" ]; then +if [[ "${build_mongot:-}" == "true" ]]; then setup_db_contrib_tool use_db_contrib_tool_mongot - bazel_args="${bazel_args} --include_mongot=True" + bazel_args="${bazel_args:-} --include_mongot=True" fi # This is hacky way to pass off build time from archive_dist_test to archive_dist_test_debug @@ -90,12 +82,12 @@ fi # We then create the symbols normally in archive_dist_test_debug. We have to force the # build-id for debugging as they will be different when -Wl,-S is passed in. # The relinked binaries should still be hash identical when stripped with strip -if [ "${skip_debug_link}" = "true" ]; then +if [[ "${skip_debug_link:-}" == "true" ]]; then export compile_variant="${compile_variant}" export version_id="${version_id}" - if [ "${task_name}" = "archive_dist_test" ]; then - task_compile_flags="${task_compile_flags} --simple_build_id=True --features=strip_debug --separate_debug=False" - if [ "${remote_link}" = "true" ]; then + if [[ "${task_name:-}" == "archive_dist_test" ]]; then + task_compile_flags="${task_compile_flags:-} --simple_build_id=True --features=strip_debug --separate_debug=False" + if [[ "${remote_link:-}" == "true" ]]; then ARCH=$(uname -m) # Remote linking is currently only supported on arm if [[ "$ARCH" == "arm64" || "$ARCH" == "aarch64" ]]; then @@ -103,66 +95,45 @@ if [ "${skip_debug_link}" = "true" ]; then fi fi fi - if [ "${task_name}" = "archive_dist_test_debug" ]; then - task_compile_flags="${task_compile_flags} --simple_build_id=True" + if [[ "${task_name:-}" == "archive_dist_test_debug" ]]; then + task_compile_flags="${task_compile_flags:-} --simple_build_id=True" fi fi -set -o pipefail - # Use `eval` to force evaluation of the environment variables in the echo statement: eval echo "Execution environment: Targets: ${targets}" -source ./evergreen/bazel_utility_functions.sh -source ./evergreen/bazel_RBE_supported.sh +BAZEL_BINARY="$(bazel_evergreen_shutils::bazel_get_binary_path)" -if [[ "${evergreen_remote_exec}" != "on" ]]; then - LOCAL_ARG="$LOCAL_ARG --jobs=auto" -fi - -BAZEL_BINARY=$(bazel_get_binary_path) - -# Timeout is set here to avoid the build hanging indefinitely, still allowing -# for retries. -TIMEOUT_CMD="" -if [ -n "${build_timeout_seconds}" ]; then - TIMEOUT_CMD="timeout ${build_timeout_seconds}" -elif [[ "${evergreen_remote_exec}" == "on" ]]; then - # Timeout remote execution runs in 60 minutes as a workaround for - # scheduling timeout bugs - TIMEOUT_CMD="timeout 3600" -fi - -if is_ppc64le; then - LOCAL_ARG="$LOCAL_ARG --jobs=48" -fi - -if is_s390x; then - LOCAL_ARG="$LOCAL_ARG --jobs=16" -fi +# Compose LOCAL_ARG and release flags +LOCAL_ARG="$(bazel_evergreen_shutils::compute_local_arg build)" # If we are doing a patch build or we are building a non-push # build on the waterfall, then we don't need the --release # flag. Otherwise, this is potentially a build that "leaves # the building", so we do want that flag. -if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then - echo "This is a non-release build." -else - LOCAL_ARG="$LOCAL_ARG --config=public-release" +LOCAL_ARG="$(bazel_evergreen_shutils::maybe_release_flag "$LOCAL_ARG")" + +# Ensure server is up and print PID +bazel_evergreen_shutils::ensure_server_and_print_pid "$BAZEL_BINARY" + +# Build flags line +ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${bazel_args:-} ${bazel_compile_flags:-} ${task_compile_flags:-} --define=MONGO_VERSION=${version} ${patch_compile_flags:-}" +echo "${ALL_FLAGS}" >.bazel_build_flags + +set +o errexit + +bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \ + build ${ALL_FLAGS} ${targets} +RET=$? + +set -o errexit + +if [[ "$RET" -eq 124 ]]; then + echo "Bazel build timed out after ${build_timeout_seconds:-} seconds." +elif [[ "$RET" != "0" ]]; then + echo "Bazel build failed after retries." fi -for i in {1..3}; do - eval ${TIMEOUT_CMD} $BAZEL_BINARY build --verbose_failures $LOCAL_ARG ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \ - --define=MONGO_VERSION=${version} ${patch_compile_flags} ${targets} 2>&1 | tee bazel_stdout.log && - RET=0 && break || RET=$? && sleep 60 - if [ $RET -eq 124 ]; then - echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..." - else - echo "Errors were found during the bazel run, here are the errors:" 1>&2 - grep "ERROR:" bazel_stdout.log 1>&2 - echo "Bazel failed to execute, retrying..." - fi - $BAZEL_BINARY shutdown -done - -exit $RET +: "${RET:=1}" +exit "${RET}" diff --git a/evergreen/bazel_evergreen_shutils.sh b/evergreen/bazel_evergreen_shutils.sh new file mode 100644 index 00000000000..3eabb03dfed --- /dev/null +++ b/evergreen/bazel_evergreen_shutils.sh @@ -0,0 +1,335 @@ +# Common helpers for CI Bazel scripts (build/run/test) + +set -o errexit +set -o pipefail + +# --- Pre-flight (assumes prelude.sh already sourced by caller) ------------- + +bazel_evergreen_shutils::activate_and_cd_src() { + cd src + set -o verbose + activate_venv +} + +# --- Distro quirks ----------------------------------------------------- + +bazel_evergreen_shutils::export_ssl_paths_if_needed() { + if [[ -f /etc/os-release ]]; then + local DISTRO + DISTRO=$(awk -F '[="]*' '/^PRETTY_NAME/ { print $2 }' " prefix or empty string. +# Prints a one-time warning to stderr if a timeout was requested but no +# timeout binary is available. Supports macOS 'gtimeout' if installed. +bazel_evergreen_shutils::timeout_prefix() { + local fallback_remote="${1:-}" # "on" = use 3600s default for remote builds + local need_timeout="" # "explicit" | "fallback" | "" + local timeout_bin="" + + # Do we want a timeout? + if [[ -n "${build_timeout_seconds:-}" ]]; then + need_timeout="explicit" + elif [[ "$fallback_remote" == "on" ]]; then + need_timeout="fallback" + fi + + # Find a timeout binary (GNU coreutils 'timeout' or macOS 'gtimeout') + if command -v timeout >/dev/null 2>&1; then + timeout_bin="timeout" + elif command -v gtimeout >/dev/null 2>&1; then + timeout_bin="gtimeout" + fi + + # If needed but unavailable, warn once and return empty + if [[ -n "$need_timeout" && -z "$timeout_bin" ]]; then + if [[ -z "${_BAZEL_EVG_TIMEOUT_WARNED:-}" ]]; then + if [[ "$need_timeout" == "explicit" ]]; then + echo "[warn] 'timeout' not found; requested ${build_timeout_seconds}s timeout will be ignored." >&2 + else + echo "[warn] 'timeout' not found; remote-build fallback timeout (3600s) will be ignored." >&2 + fi + # Helpful hint for macOS users + if bazel_evergreen_shutils::is_macos; then + echo "[hint] On macOS, install coreutils: 'brew install coreutils' (provides 'gtimeout')." >&2 + fi + _BAZEL_EVG_TIMEOUT_WARNED=1 + fi + echo "" # no timeout prefix + return 0 + fi + + # Produce the prefix if we have a binary + if [[ -n "$timeout_bin" ]]; then + if [[ "$need_timeout" == "explicit" ]]; then + echo "$timeout_bin ${build_timeout_seconds}" + elif [[ "$need_timeout" == "fallback" ]]; then + echo "$timeout_bin 3600" + else + echo "" + fi + else + echo "" + fi +} + +# --- Bazel server lifecycle & OOM-detect retry ----------------------------- + +bazel_evergreen_shutils::bazel_output_base() { + local BAZEL_BINARY="$1" + "$BAZEL_BINARY" info output_base 2>/dev/null +} + +bazel_evergreen_shutils::bazel_pidfile_path() { + local BAZEL_BINARY="$1" + local ob + ob="$(bazel_evergreen_shutils::bazel_output_base "$BAZEL_BINARY")" || return 1 + echo "${ob}/server/server.pid.txt" +} + +bazel_evergreen_shutils::is_bazel_server_running() { + local BAZEL_BINARY="$1" + local pf pid + pf="$(bazel_evergreen_shutils::bazel_pidfile_path "$BAZEL_BINARY")" || return 1 + [[ -f "$pf" ]] || return 1 + pid="$(cat "$pf" 2>/dev/null || true)" + [[ -n "$pid" ]] || return 1 + if kill -0 "$pid" 2>/dev/null; then + return 0 + else + return 1 + fi +} + +bazel_evergreen_shutils::print_bazel_server_pid() { + local BAZEL_BINARY="$1" + local pf pid + pf="$(bazel_evergreen_shutils::bazel_pidfile_path "$BAZEL_BINARY")" || { + echo "Bazel server pidfile not found (output_base: $(bazel_evergreen_shutils::bazel_output_base "$BAZEL_BINARY" 2>/dev/null || true))" + return 0 + } + if [[ -f "$pf" ]]; then + pid="$(cat "$pf" 2>/dev/null || true)" + echo "Bazel server pidfile: $pf (PID=${pid:-unknown})" + else + echo "Bazel server pidfile not found yet (output_base: $(bazel_evergreen_shutils::bazel_output_base "$BAZEL_BINARY" 2>/dev/null || true))" + fi +} + +# Starts server (if needed) and prints PID. Safe to call multiple times. +bazel_evergreen_shutils::ensure_server_and_print_pid() { + local BAZEL_BINARY="$1" + _IGN=$("$BAZEL_BINARY" info >/dev/null 2>&1 || true) + bazel_evergreen_shutils::print_bazel_server_pid "$BAZEL_BINARY" +} + +# Generic retry wrapper: +# $1: attempts +# $3: bazel binary +# $4..: full bazel subcommand + args (e.g. "build --verbose_failures ...") +# Special handling: +# - exit 124 -> timeout +# - server death (pid missing) -> restart, then retry +# Returns with global RET set. +bazel_evergreen_shutils::retry_bazel_cmd() { + local attempts="$1" + shift + local BAZEL_BINARY="$1" + shift + + local timeout_str="$(bazel_evergreen_shutils::timeout_prefix "${evergreen_remote_exec:-}")" + + # Everything else is the Bazel subcommand + flags (and possibly redirections/pipes). + # We *intentionally* keep it as raw words and reassemble to a single string for eval. + local raw_rest=("$@") + + # Once we detect an OOM/server-death, we enable the guard for subsequent attempts. + local use_oom_guard=false + local -r OOM_GUARD_FLAG='--local_resources=HOST_CPUS*.5' + + # Helper: does the current command string already include a local_resources flag? + _cmd_has_local_resources() { + [[ "$1" == *"--local_resources"* ]] + } + + local RET=1 + + for i in $(seq 1 "$attempts"); do + echo "Attempt ${i}/${attempts}…" >&2 + + # Ensure/refresh server & pid before we run (helps produce a fresh pidfile too). + if ! bazel_evergreen_shutils::is_bazel_server_running "$BAZEL_BINARY"; then + echo "[retry ${i}] Bazel server not running (likely OOM/killed); restarting…" >&2 + "$BAZEL_BINARY" info >/dev/null 2>&1 || true + bazel_evergreen_shutils::print_bazel_server_pid "$BAZEL_BINARY" >&2 + fi + + # Reassemble the caller’s words into a single command string for eval. + # We deliberately do *not* try to be clever here—this restores legacy behavior + # where quoted pieces inside variables (e.g., --base_dir="..") are honored by the shell. + local cmd="\"$BAZEL_BINARY\"" + local w + for w in "${raw_rest[@]}"; do + cmd+=" $w" + done + + # If OOM guard is enabled and not already present, append it. + # (Safe with eval; if the caller added redirections earlier, this is still just an arg.) + if $use_oom_guard && ! _cmd_has_local_resources "$cmd"; then + echo "[retry ${i}] Applying OOM guard: ${OOM_GUARD_FLAG}" >&2 + cmd+=" ${OOM_GUARD_FLAG}" + fi + + # Prefix timeout, if any. + if [[ -n "$timeout_str" ]]; then + cmd="${timeout_str} ${cmd}" + fi + + # Run it. + # NOTE: We *do not* add any redirections here; caller controls logging completely. + if eval "$cmd"; then + RET=0 + break + else + RET=$? + fi + + # Classify failure & decide on guard for next attempt. + [[ $RET -eq 124 ]] && echo "Bazel timed out." >&2 + + if ! bazel_evergreen_shutils::is_bazel_server_running "$BAZEL_BINARY"; then + echo "[retry ${i}] Bazel server down (OOM/killed). Enabling OOM guard for next attempt and restarting…" >&2 + use_oom_guard=true + "$BAZEL_BINARY" shutdown || true + "$BAZEL_BINARY" info >/dev/null 2>&1 || true + bazel_evergreen_shutils::print_bazel_server_pid "$BAZEL_BINARY" >&2 + else + echo "Bazel failed (exit=$RET); restarting server before retry…" >&2 + "$BAZEL_BINARY" shutdown || true + fi + + sleep 60 + done + + return "$RET" + +} + +# --- Test helpers ---------------------------------------------------------- + +# Multiplies test timeout on slow arches and appends to bazel_args +bazel_evergreen_shutils::maybe_scale_test_timeout_and_append() { + if [[ -n "${test_timeout_sec:-}" ]]; then + local scaled="$test_timeout_sec" + if bazel_evergreen_shutils::is_s390x_or_ppc64le; then + scaled=$((test_timeout_sec * 4)) + fi + bazel_args="${bazel_args:-} --test_timeout=${scaled}" + fi +} diff --git a/evergreen/bazel_run.sh b/evergreen/bazel_run.sh index b35fe005ae3..eb14676650d 100644 --- a/evergreen/bazel_run.sh +++ b/evergreen/bazel_run.sh @@ -7,73 +7,54 @@ # * ${env} - Env variable string to set (ex. ENV_VAR_ABC=123) # * ${redact_args} - If set, redact the args in the report -# Needed for evergreen scripts that use evergreen expansions and utility methods. DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" -cd src - set -o errexit set -o verbose +set -o pipefail -activate_venv +. "$DIR/bazel_evergreen_shutils.sh" + +bazel_evergreen_shutils::activate_and_cd_src +bazel_evergreen_shutils::export_ssl_paths_if_needed # Use `eval` to force evaluation of the environment variables in the echo statement: eval echo "Execution environment: Args: ${args} Target: ${target} Env: ${env} redact_args: ${redact_args}" -source ./evergreen/bazel_utility_functions.sh -source ./evergreen/bazel_RBE_supported.sh +BAZEL_BINARY="$(bazel_evergreen_shutils::bazel_get_binary_path)" -if bazel_rbe_supported; then - LOCAL_ARG="" -else - LOCAL_ARG="--config=local" -fi - -if [[ "${evergreen_remote_exec}" != "on" ]]; then - LOCAL_ARG="--config=local" -fi - -BAZEL_BINARY=$(bazel_get_binary_path) - -# AL2 stores certs in a nonstandard location -if [[ -f /etc/os-release ]]; then - DISTRO=$(awk -F '[="]*' '/^PRETTY_NAME/ { print $2 }' >bazel-invocation.txt +INVOCATION_WITH_REDACTION="${target}" +if [[ -z "${redact_args:-}" ]]; then + INVOCATION_WITH_REDACTION+=" ${args}" +fi -# Run bazel command, retrying up to five times -MAX_ATTEMPTS=5 -for ((i = 1; i <= $MAX_ATTEMPTS; i++)); do - eval $env $BAZEL_BINARY run --verbose_failures ${LOCAL_ARG} ${target} ${args} >>bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 10 - if [ $i -lt $MAX_ATTEMPTS ]; then echo "Bazel failed to execute, retrying ($(($i + 1)) of $MAX_ATTEMPTS attempts)... " >>bazel_output.log 2>&1; fi - $BAZEL_BINARY shutdown -done +# Record invocation +echo "bazel run --verbose_failures ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >bazel-invocation.txt -$python ./buildscripts/simple_report.py --test-name "bazel run ${INVOCATION_WITH_REDACTION}" --log-file bazel_output.log --exit-code $RET -exit $RET +# capture exit code +set +o errexit + +bazel_evergreen_shutils::retry_bazel_cmd 5 "$BAZEL_BINARY" \ + run --verbose_failures ${LOCAL_ARG} ${target} ${args} 2>&1 | tee -a bazel_output.log +RET=${PIPESTATUS[0]} +: "${RET:=1}" +set -o errexit + +# Report +$python ./buildscripts/simple_report.py \ + --test-name "bazel run ${INVOCATION_WITH_REDACTION}" \ + --log-file bazel_output.log \ + --exit-code "${RET}" + +exit "${RET}" diff --git a/evergreen/bazel_test.sh b/evergreen/bazel_test.sh index 5e5dd380fce..d533e019e9b 100644 --- a/evergreen/bazel_test.sh +++ b/evergreen/bazel_test.sh @@ -8,99 +8,65 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" -cd src - set -o errexit set -o verbose - -activate_venv - set -o pipefail +. "$DIR/bazel_evergreen_shutils.sh" + +bazel_evergreen_shutils::activate_and_cd_src +bazel_evergreen_shutils::export_ssl_paths_if_needed + # Use `eval` to force evaluation of the environment variables in the echo statement: eval echo "Execution environment: Targets: ${targets}" -source ./evergreen/bazel_utility_functions.sh -source ./evergreen/bazel_RBE_supported.sh +BAZEL_BINARY="$(bazel_evergreen_shutils::bazel_get_binary_path)" -LOCAL_ARG="" -if [[ "${evergreen_remote_exec}" != "on" ]]; then - LOCAL_ARG="$LOCAL_ARG --jobs=auto" -elif [[ "${task_name}" == "unit_tests" ]]; then - LOCAL_ARG="$LOCAL_ARG --config=remote_test" -fi - -BAZEL_BINARY=$(bazel_get_binary_path) - -# Timeout is set here to avoid the build hanging indefinitely, still allowing -# for retries. -TIMEOUT_CMD="" -if [ -n "${build_timeout_seconds}" ]; then - TIMEOUT_CMD="timeout ${build_timeout_seconds}" -fi - -if is_ppc64le; then - LOCAL_ARG="$LOCAL_ARG --jobs=48" -fi - -if is_s390x; then - LOCAL_ARG="$LOCAL_ARG --jobs=16" -fi +# Mode-specific LOCAL_ARG and release flag +LOCAL_ARG="$(bazel_evergreen_shutils::compute_local_arg test)" # If we are doing a patch build or we are building a non-push # build on the waterfall, then we don't need the --release # flag. Otherwise, this is potentially a build that "leaves # the building", so we do want that flag. -if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then - echo "This is a non-release build." -else - LOCAL_ARG="$LOCAL_ARG --config=public-release" -fi +LOCAL_ARG="$(bazel_evergreen_shutils::maybe_release_flag "$LOCAL_ARG")" -if [ -n "${test_timeout_sec}" ]; then - # s390x and ppc64le often run slower than other architectures - if is_s390x_or_ppc64le; then - test_timeout_sec=$(($test_timeout_sec * 4)) - fi - bazel_args="${bazel_args} --test_timeout=${test_timeout_sec}" -fi +# Possibly scale test timeout and append to bazel_args +bazel_evergreen_shutils::maybe_scale_test_timeout_and_append -ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} --define=MONGO_VERSION=${version} ${patch_compile_flags}" -echo ${ALL_FLAGS} >.bazel_build_flags +# Build the shared flags and persist the --config subset +ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${bazel_args:-} ${bazel_compile_flags:-} ${task_compile_flags:-} --define=MONGO_VERSION=${version} ${patch_compile_flags:-}" +echo "${ALL_FLAGS}" >.bazel_build_flags +# to capture exit codes set +o errexit -# Retry the build since it's deterministic and may fail due to transient issues. -for i in {1..3}; do - eval ${TIMEOUT_CMD} ${BAZEL_BINARY} build ${ALL_FLAGS} ${targets} && RET=0 && break || RET=$? && sleep 1 - if [ $RET -eq 124 ]; then - echo "Bazel build timed out after ${build_timeout_seconds} seconds, retrying..." - else - echo "Bazel build failed, retrying..." - fi - $BAZEL_BINARY shutdown -done +# Build then test with retries. +bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \ + build ${ALL_FLAGS} ${targets} +RET=$? -for i in {1..3}; do - eval ${TIMEOUT_CMD} ${BAZEL_BINARY} test ${ALL_FLAGS} ${targets} 2>&1 | tee bazel_stdout.log && - RET=0 && break || RET=$? && sleep 1 - if [ $RET -eq 124 ]; then - echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..." - else - echo "Errors were found during the bazel test, failing the execution" - break +if [[ "$RET" == "0" ]]; then + + bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \ + test ${ALL_FLAGS} ${targets} + RET=$? + + if [[ "$RET" -eq 124 ]]; then + echo "Bazel timed out after ${build_timeout_seconds:-} seconds." + elif [[ "$RET" != "0" ]]; then + echo "Errors were found during bazel test, failing the execution" fi - $BAZEL_BINARY shutdown -done +fi set -o errexit -if [[ $RET != 0 ]]; then - # The --config flag needs to stay consistent between invocations to avoid evicting the previous results. - # Strip out anything that isn't a --config flag that could interfere with the run command. - CONFIG_FLAGS=$(echo "${ALL_FLAGS}" | tr ' ' '\n' | grep -- '--config' | tr '\n' ' ') - - eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:gather_failed_unittests +# The --config flag needs to stay consistent between invocations to avoid evicting the previous results. +# Strip out anything that isn't a --config flag that could interfere with the run command. +if [[ "$RET" != "0" ]]; then + CONFIG_FLAGS="$(bazel_evergreen_shutils::extract_config_flags "${ALL_FLAGS}")" + eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:gather_failed_unittests || true fi -exit $RET +: "${RET:=1}" +exit "${RET}" diff --git a/evergreen/bazel_utility_functions.sh b/evergreen/bazel_utility_functions.sh deleted file mode 100755 index d855a80493f..00000000000 --- a/evergreen/bazel_utility_functions.sh +++ /dev/null @@ -1,30 +0,0 @@ -function is_macos() { - local -r os="$(uname -s | tr '[:upper:]' '[:lower:]')" - [[ "${os}" == "darwin" ]] && return 0 || return 1 -} - -function is_ppc64le() { - local -r arch="$(uname -m)" - [[ "${arch}" == "ppc64le" || "${arch}" == "ppc64" || "${arch}" == "ppc" ]] && return 0 || return 1 -} - -function is_s390x() { - local -r arch="$(uname -m)" - [[ "${arch}" == "s390x" || "${arch}" == "s390" ]] && return 0 || return 1 -} - -function is_s390x_or_ppc64le() { - (is_ppc64le || is_s390x) && return 0 || return 1 -} - -function bazel_get_binary_path() { - if is_macos; then - echo "bazel" - elif is_s390x_or_ppc64le || - grep -q "ID=debian" /etc/os-release || - grep -q 'ID="sles"' /etc/os-release; then - echo "bazel/bazelisk.py" - else - echo "bazel" - fi -} diff --git a/evergreen/compile_ninja.sh b/evergreen/compile_ninja.sh deleted file mode 100644 index a216e08ad8b..00000000000 --- a/evergreen/compile_ninja.sh +++ /dev/null @@ -1,12 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -cd src - -activate_venv -if [ "Windows_NT" = "$OS" ]; then - vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/" - cd "$vcvars" && cmd /K "vcvarsall.bat amd64 && cd ${workdir}\src" -fi -python -m pip install ninja -ninja install-core diff --git a/evergreen/get_all_resmoke_suite_configs.sh b/evergreen/get_all_resmoke_suite_configs.sh index 3ce4f690c55..e0877a63676 100644 --- a/evergreen/get_all_resmoke_suite_configs.sh +++ b/evergreen/get_all_resmoke_suite_configs.sh @@ -12,8 +12,8 @@ cd src set -o errexit set -o verbose -source ./evergreen/bazel_utility_functions.sh -BAZEL_BINARY=$(bazel_get_binary_path) +source ./evergreen/bazel_evergreen_shutils.sh +BAZEL_BINARY=$(bazel_evergreen_shutils::bazel_get_binary_path) # Queries all resmoke_config targets: kind(resmoke_config, //...) # and outputs YAML key-value pair created by the starlark expression for each target. diff --git a/evergreen/get_resmoke_suite_config.sh b/evergreen/get_resmoke_suite_config.sh index 173f5b0d376..dfe5a1f0bdd 100644 --- a/evergreen/get_resmoke_suite_config.sh +++ b/evergreen/get_resmoke_suite_config.sh @@ -14,8 +14,8 @@ cd src set -o errexit set -o verbose -source ./evergreen/bazel_utility_functions.sh -BAZEL_BINARY=$(bazel_get_binary_path) +source ./evergreen/bazel_evergreen_shutils.sh +BAZEL_BINARY=$(bazel_evergreen_shutils::bazel_get_binary_path) echo "suite_config: $(${BAZEL_BINARY} cquery ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \ --define=MONGO_VERSION=${version} ${patch_compile_flags} ${suite}_config --output files)" >suite_config_expansion.yml diff --git a/evergreen/resmoke_tests_execute_bazel.sh b/evergreen/resmoke_tests_execute_bazel.sh index 67f5248833c..c42682174d6 100644 --- a/evergreen/resmoke_tests_execute_bazel.sh +++ b/evergreen/resmoke_tests_execute_bazel.sh @@ -16,9 +16,9 @@ set -o verbose activate_venv -source ./evergreen/bazel_utility_functions.sh +source ./evergreen/bazel_evergreen_shutils.sh -BAZEL_BINARY=$(bazel_get_binary_path) +BAZEL_BINARY=$(bazel_evergreen_shutils::bazel_get_binary_path) # Timeout is set here to avoid the build hanging indefinitely, still allowing # for retries.