mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-113201 bazel retry logic in CI should not retry on regular fail in some cases (#43361) 

GitOrigin-RevId: 5cfd14aa58b6f3f547a83de29b65a169d96184e5
This commit is contained in:
Daniel Moody 2025-10-30 14:21:37 -05:00 committed by MongoDB Bot
parent 3aca524542
commit 03e15a5a11
4 changed files with 13 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
evergreen/bazel_compile.sh
View File

@ -126,6 +126,7 @@ echo "bazel build ${ALL_FLAGS} ${targets}" >.bazel_build_invocation
set +o errexit set +o errexit
export RETRY_ON_FAIL=1
bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \ bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \
build ${ALL_FLAGS} ${targets} build ${ALL_FLAGS} ${targets}
RET=$? RET=$?

14
evergreen/bazel_evergreen_shutils.sh
View File

@ -308,18 +308,22 @@ bazel_evergreen_shutils::retry_bazel_cmd() {
RET=$? RET=$?
fi fi
# Classify failure & decide on guard for next attempt.
[[ $RET -eq 124 ]] && echo "Bazel timed out." >&2
if ! bazel_evergreen_shutils::is_bazel_server_running "$BAZEL_BINARY"; then if ! bazel_evergreen_shutils::is_bazel_server_running "$BAZEL_BINARY"; then
echo "[retry ${i}] Bazel server down (OOM/killed). Enabling OOM guard for next attempt and restarting…" >&2 echo "[retry ${i}] Bazel server down (OOM/killed). Enabling OOM guard for next attempt and restarting…" >&2
use_oom_guard=true use_oom_guard=true
"$BAZEL_BINARY" shutdown || true "$BAZEL_BINARY" shutdown || true
"$BAZEL_BINARY" info >/dev/null 2>&1 || true "$BAZEL_BINARY" info >/dev/null 2>&1 || true
bazel_evergreen_shutils::print_bazel_server_pid "$BAZEL_BINARY" >&2 bazel_evergreen_shutils::print_bazel_server_pid "$BAZEL_BINARY" >&2
else elif [[ $RET -eq 124 ]]; then
echo "Bazel failed (exit=$RET); restarting server before retry…" >&2 echo "Bazel timed out." >&2
"$BAZEL_BINARY" shutdown || true "$BAZEL_BINARY" shutdown || true
else
if [[ ${RETRY_ON_FAIL:-0} -eq 1 ]]; then
echo "Bazel failed (exit=$RET); restarting server before retry..." >&2
"$BAZEL_BINARY" shutdown || true
else
break
fi
fi fi
sleep 60 sleep 60

2
evergreen/bazel_run.sh
View File

@ -44,7 +44,7 @@ echo "bazel run --verbose_failures ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >b
# capture exit code # capture exit code
set +o errexit set +o errexit
export RETRY_ON_FAIL=0
bazel_evergreen_shutils::retry_bazel_cmd 5 "$BAZEL_BINARY" \ bazel_evergreen_shutils::retry_bazel_cmd 5 "$BAZEL_BINARY" \
run --verbose_failures ${LOCAL_ARG} ${target} ${args} 2>&1 | tee -a bazel_output.log run --verbose_failures ${LOCAL_ARG} ${target} ${args} 2>&1 | tee -a bazel_output.log
RET=${PIPESTATUS[0]} RET=${PIPESTATUS[0]}

3
evergreen/bazel_test.sh
View File

@ -42,12 +42,13 @@ echo "${ALL_FLAGS}" >.bazel_build_flags
set +o errexit set +o errexit
# Build then test with retries. # Build then test with retries.
export RETRY_ON_FAIL=1
bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \ bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \
build ${ALL_FLAGS} ${targets} build ${ALL_FLAGS} ${targets}
RET=$? RET=$?
if [[ "$RET" == "0" ]]; then if [[ "$RET" == "0" ]]; then
export RETRY_ON_FAIL=0
bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \ bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \
test ${ALL_FLAGS} ${targets} test ${ALL_FLAGS} ${targets}
RET=$? RET=$?