SERVER-113201 bazel retry logic in CI should not retry on regular fail in some cases (#43361)

GitOrigin-RevId: 5cfd14aa58b6f3f547a83de29b65a169d96184e5

evergreen/bazel_compile.sh

@@ -126,6 +126,7 @@ echo "bazel build ${ALL_FLAGS} ${targets}" >.bazel_build_invocation
 
 set +o errexit
 
+export RETRY_ON_FAIL=1
 bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \
     build ${ALL_FLAGS} ${targets}
 RET=$?

evergreen/bazel_evergreen_shutils.sh

@@ -308,18 +308,22 @@ bazel_evergreen_shutils::retry_bazel_cmd() {
             RET=$?
         fi
 
-        # Classify failure & decide on guard for next attempt.
-        [[ $RET -eq 124 ]] && echo "Bazel timed out." >&2
-
         if ! bazel_evergreen_shutils::is_bazel_server_running "$BAZEL_BINARY"; then
             echo "[retry ${i}] Bazel server down (OOM/killed). Enabling OOM guard for next attempt and restarting…" >&2
             use_oom_guard=true
             "$BAZEL_BINARY" shutdown || true
             "$BAZEL_BINARY" info >/dev/null 2>&1 || true
             bazel_evergreen_shutils::print_bazel_server_pid "$BAZEL_BINARY" >&2
-        else
-            echo "Bazel failed (exit=$RET); restarting server before retry…" >&2
+        elif [[ $RET -eq 124 ]]; then
+            echo "Bazel timed out." >&2
             "$BAZEL_BINARY" shutdown || true
+        else
+            if [[ ${RETRY_ON_FAIL:-0} -eq 1 ]]; then
+                echo "Bazel failed (exit=$RET); restarting server before retry..." >&2
+                "$BAZEL_BINARY" shutdown || true
+            else
+                break
+            fi
         fi
 
         sleep 60

evergreen/bazel_run.sh

@@ -44,7 +44,7 @@ echo "bazel run --verbose_failures ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >b
 
 # capture exit code
 set +o errexit
-
+export RETRY_ON_FAIL=0
 bazel_evergreen_shutils::retry_bazel_cmd 5 "$BAZEL_BINARY" \
     run --verbose_failures ${LOCAL_ARG} ${target} ${args} 2>&1 | tee -a bazel_output.log
 RET=${PIPESTATUS[0]}

evergreen/bazel_test.sh

@@ -42,12 +42,13 @@ echo "${ALL_FLAGS}" >.bazel_build_flags
 set +o errexit
 
 # Build then test with retries.
+export RETRY_ON_FAIL=1
 bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \
     build ${ALL_FLAGS} ${targets}
 RET=$?
 
 if [[ "$RET" == "0" ]]; then
-
+    export RETRY_ON_FAIL=0
     bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \
         test ${ALL_FLAGS} ${targets}
     RET=$?