SERVER-81038 Self host bazelisk and retry downloading on exception (#19327)

GitOrigin-RevId: db5225317044f7f2c73fc7453ddedeb4105606dd

buildfarm/toolchain.dockerfile

@@ -14,7 +14,6 @@ RUN git clone -b 2.3.1 https://github.com/bazelbuild/bazel-buildfarm.git
 # Switch into the cloned Buildfarm repository
 WORKDIR /bazel-buildfarm
 
-# TODO(SERVER-81038): remove once bazel/bazelisk is self-hosted.
 # Obtain Bazelisk and make it executable
 RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-arm64 -O bazelisk && chmod +x bazelisk
 

buildscripts/libdeps/libdeps/analyzer.py

@@ -540,7 +540,6 @@ class BazelConversionCandidates(Analyzer):
 
         # Note: //... is the bazel catch-all for referencing all targets in that directory. For
         # example, //src/... will expand to include all targets under //src/.
-        # TODO(SERVER-81038): remove /tmp/ prefix once bazel/bazelisk is self-hosted.
         proc = subprocess.run(["/tmp/bazelisk", "query", "//..."], capture_output=True, text=True,
                               check=True)
 

docs/bazel.md

@@ -16,7 +16,7 @@ To generate and install the engflow credentials:
 
 To perform a Bazel build and _bypass_ SCons:
 
--   Install Bazelisk: `curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-arm64 --output /tmp/bazelisk && chmod +x /tmp/bazelisk`
+-   Install Bazelisk: `curl -L https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-linux-arm64 --output /tmp/bazelisk && chmod +x /tmp/bazelisk`
 -   Build the Bazel-compatible target: `/tmp/bazelisk build --verbose_failures  src/mongo/db/commands:fsync_locked`
 
 To perform a Bazel build using a local Buildfarm (to test remote execution capability):

etc/evergreen_yml_components/tasks/compile_tasks.yml

@@ -216,8 +216,6 @@ tasks:
   - name: version_expansions_gen
     variant: generate-tasks-for-version
   commands:
-  # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
-  - func: "fetch bazel"
   - func: "scons compile"
     vars:
       targets: >-
@@ -241,8 +239,6 @@ tasks:
   - name: version_expansions_gen
     variant: generate-tasks-for-version
   commands:
-  # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
-  - func: "fetch bazel"
   - func: "scons compile"
     vars:
       targets: >-
@@ -269,7 +265,6 @@ tasks:
   - name: version_expansions_gen
     variant: generate-tasks-for-version
   commands:
-  # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
   - func: "fetch bazel"
   - func: "bazel compile (gcc)"
     vars:
@@ -285,8 +280,6 @@ tasks:
   - name: version_expansions_gen
     variant: generate-tasks-for-version
   commands:
-  # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
-  - func: "fetch bazel"
   - func: "scons compile"
     vars:
       targets: >-
@@ -310,8 +303,6 @@ tasks:
   - name: version_expansions_gen
     variant: generate-tasks-for-version
   commands:
-  # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
-  - func: "fetch bazel"
   - func: "scons compile"
     vars:
       targets: >-
@@ -336,8 +327,6 @@ tasks:
   - name: version_expansions_gen
     variant: generate-tasks-for-version
   commands:
-  # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
-  - func: "fetch bazel"
   - func: "scons compile"
     vars:
       targets: >-
@@ -361,8 +350,6 @@ tasks:
   - name: version_expansions_gen
     variant: generate-tasks-for-version
   commands:
-  # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
-  - func: "fetch bazel"
   - func: "scons compile"
     vars:
       targets: >-
@@ -388,7 +375,6 @@ tasks:
   - name: version_expansions_gen
     variant: generate-tasks-for-version
   commands:
-  # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
   - func: "fetch bazel"
   - func: "bazel run"
     vars:
@@ -403,7 +389,6 @@ tasks:
   - name: version_expansions_gen
     variant: generate-tasks-for-version
   commands:
-  # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
   - func: "fetch bazel"
   - func: "bazel run"
     vars:

evergreen/get_bazelisk.sh

@@ -33,8 +33,7 @@ if [[ $ARCH == "ppc64le" ]]; then
   REMOTE_PATH=https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-${ARCH}
   LOCAL_PATH=$TMPDIR/bazel
 else
-  # TODO(SERVER-81038): remove once bazel/bazelisk is self-hosted.
+  REMOTE_PATH=https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-${OS}-${ARCH}${EXT}
-  REMOTE_PATH=https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-${OS}-${ARCH}${EXT}
   LOCAL_PATH=$TMPDIR/bazelisk
 fi
 

site_scons/site_tools/integrate_bazel.py

@@ -15,6 +15,7 @@ import time
 from typing import List, Dict, Set, Tuple, Any
 import urllib.request
 import requests
+from retry import retry
 import sys
 import socket
 
@@ -47,6 +48,16 @@ _SANITIZER_MAP = {
 _S3_HASH_MAPPING = {
     "https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-ppc64le":
         "dd21c75817533ff601bf797e64f0eb2f7f6b813af26c829f0bda30e328caef46",
+    "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-darwin-amd64":
+        "f2ba5f721a995b54bab68c6b76a340719888aa740310e634771086b6d1528ecd",
+    "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-darwin-arm64":
+        "69fa21cd2ccffc2f0970c21aa3615484ba89e3553ecce1233a9d8ad9570d170e",
+    "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-linux-amd64":
+        "d28b588ac0916abd6bf02defb5433f6eddf7cba35ffa808eabb65a44aab226f7",
+    "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-linux-arm64":
+        "861a16ba9979613e70bd3d2f9d9ab5e3b59fe79471c5753acdc9c431ab6c9d94",
+    "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-windows-amd64.exe":
+        "d04555245a99dfb628e33da24e2b9198beb8f46d7e7661c313eb045f6a59f5e4",
 }
 
 
@@ -486,6 +497,11 @@ def generate_bazel_info_for_ninja(env: SCons.Environment.Environment) -> None:
     env["NINJA_BAZEL_INPUTS"] = ninja_bazel_ins
 
 
+@retry(tries=5, delay=3)
+def download_path_with_retry(*args, **kwargs):
+    urllib.request.urlretrieve(*args, **kwargs)
+
+
 def sha256_file(filename: str) -> str:
     sha256_hash = hashlib.sha256()
     with open(filename, "rb") as f:
@@ -544,20 +560,19 @@ def generate(env: SCons.Environment.Environment) -> None:
             bazel_bin_dir, "bazel") if normalized_arch in ["ppc64le"] else os.path.join(
                 bazel_bin_dir, "bazelisk")
 
-        # TODO(SERVER-81038): remove once bazel/bazelisk is self-hosted.
         if not os.path.exists(bazel_executable):
             print(f"Downloading {bazel_executable}...")
             # TODO(SERVER-86050): remove the branch once bazelisk is built on s390x & ppc64le
             if normalized_arch in ["ppc64le"]:
                 s3_path = f"https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-{normalized_arch}"
-                urllib.request.urlretrieve(s3_path, bazel_executable)
-                verify_s3_hash(s3_path, bazel_executable)
             else:
                 ext = ".exe" if normalized_os == "windows" else ""
                 os_str = normalized_os.replace("macos", "darwin")
-                urllib.request.urlretrieve(
+                s3_path = f"https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-{os_str}-{normalized_arch}{ext}"
-                    f"https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-{os_str}-{normalized_arch}{ext}",
+
-                    bazel_executable)
+            download_path_with_retry(s3_path, bazel_executable)
+            verify_s3_hash(s3_path, bazel_executable)
+
             print(f"Downloaded {bazel_executable}")
             os.chmod(bazel_executable, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
         else: