diff --git a/evergreen/bazel_compile.sh b/evergreen/bazel_compile.sh index 27094ba0730..4f0068fb824 100755 --- a/evergreen/bazel_compile.sh +++ b/evergreen/bazel_compile.sh @@ -126,6 +126,7 @@ echo "bazel build ${ALL_FLAGS} ${targets}" >.bazel_build_invocation set +o errexit +export RETRY_ON_FAIL=1 bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \ build ${ALL_FLAGS} ${targets} RET=$? diff --git a/evergreen/bazel_evergreen_shutils.sh b/evergreen/bazel_evergreen_shutils.sh index d9b752b5f64..7c27c616b95 100644 --- a/evergreen/bazel_evergreen_shutils.sh +++ b/evergreen/bazel_evergreen_shutils.sh @@ -308,18 +308,22 @@ bazel_evergreen_shutils::retry_bazel_cmd() { RET=$? fi - # Classify failure & decide on guard for next attempt. - [[ $RET -eq 124 ]] && echo "Bazel timed out." >&2 - if ! bazel_evergreen_shutils::is_bazel_server_running "$BAZEL_BINARY"; then echo "[retry ${i}] Bazel server down (OOM/killed). Enabling OOM guard for next attempt and restarting…" >&2 use_oom_guard=true "$BAZEL_BINARY" shutdown || true "$BAZEL_BINARY" info >/dev/null 2>&1 || true bazel_evergreen_shutils::print_bazel_server_pid "$BAZEL_BINARY" >&2 - else - echo "Bazel failed (exit=$RET); restarting server before retry…" >&2 + elif [[ $RET -eq 124 ]]; then + echo "Bazel timed out." >&2 "$BAZEL_BINARY" shutdown || true + else + if [[ ${RETRY_ON_FAIL:-0} -eq 1 ]]; then + echo "Bazel failed (exit=$RET); restarting server before retry..." >&2 + "$BAZEL_BINARY" shutdown || true + else + break + fi fi sleep 60 diff --git a/evergreen/bazel_run.sh b/evergreen/bazel_run.sh index eb14676650d..41976b63cb8 100644 --- a/evergreen/bazel_run.sh +++ b/evergreen/bazel_run.sh @@ -44,7 +44,7 @@ echo "bazel run --verbose_failures ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >b # capture exit code set +o errexit - +export RETRY_ON_FAIL=0 bazel_evergreen_shutils::retry_bazel_cmd 5 "$BAZEL_BINARY" \ run --verbose_failures ${LOCAL_ARG} ${target} ${args} 2>&1 | tee -a bazel_output.log RET=${PIPESTATUS[0]} diff --git a/evergreen/bazel_test.sh b/evergreen/bazel_test.sh index 145c178114e..cbe82801231 100644 --- a/evergreen/bazel_test.sh +++ b/evergreen/bazel_test.sh @@ -42,12 +42,13 @@ echo "${ALL_FLAGS}" >.bazel_build_flags set +o errexit # Build then test with retries. +export RETRY_ON_FAIL=1 bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \ build ${ALL_FLAGS} ${targets} RET=$? if [[ "$RET" == "0" ]]; then - + export RETRY_ON_FAIL=0 bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \ test ${ALL_FLAGS} ${targets} RET=$?