mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-81038 Self host bazelisk and retry downloading on exception (#19327) 

GitOrigin-RevId: db5225317044f7f2c73fc7453ddedeb4105606dd
This commit is contained in:
Zack Winter 2024-02-28 16:18:13 -08:00 committed by MongoDB Bot
parent 4d46168300
commit 9b769b44fa
6 changed files with 23 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
buildfarm/toolchain.dockerfile
View File

@ -14,7 +14,6 @@ RUN git clone -b 2.3.1 https://github.com/bazelbuild/bazel-buildfarm.git
# Switch into the cloned Buildfarm repository
WORKDIR /bazel-buildfarm
# TODO(SERVER-81038): remove once bazel/bazelisk is self-hosted.
# Obtain Bazelisk and make it executable
RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-arm64 -O bazelisk && chmod +x bazelisk

1
buildscripts/libdeps/libdeps/analyzer.py
View File

@ -540,7 +540,6 @@ class BazelConversionCandidates(Analyzer):
# Note: //... is the bazel catch-all for referencing all targets in that directory. For
# example, //src/... will expand to include all targets under //src/.
# TODO(SERVER-81038): remove /tmp/ prefix once bazel/bazelisk is self-hosted.
proc = subprocess.run(["/tmp/bazelisk", "query", "//..."], capture_output=True, text=True,
check=True)

2
docs/bazel.md
View File

@ -16,7 +16,7 @@ To generate and install the engflow credentials:
To perform a Bazel build and _bypass_ SCons:
- Install Bazelisk: `curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-arm64 --output /tmp/bazelisk && chmod +x /tmp/bazelisk`
- Install Bazelisk: `curl -L https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-linux-arm64 --output /tmp/bazelisk && chmod +x /tmp/bazelisk`
- Build the Bazel-compatible target: `/tmp/bazelisk build --verbose_failures src/mongo/db/commands:fsync_locked`
To perform a Bazel build using a local Buildfarm (to test remote execution capability):

15
etc/evergreen_yml_components/tasks/compile_tasks.yml
View File

@ -216,8 +216,6 @@ tasks:
- name: version_expansions_gen
variant: generate-tasks-for-version
commands:
# TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
- func: "fetch bazel"
- func: "scons compile"
vars:
targets: >-
@ -241,8 +239,6 @@ tasks:
- name: version_expansions_gen
variant: generate-tasks-for-version
commands:
# TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
- func: "fetch bazel"
- func: "scons compile"
vars:
targets: >-
@ -269,7 +265,6 @@ tasks:
- name: version_expansions_gen
variant: generate-tasks-for-version
commands:
# TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
- func: "fetch bazel"
- func: "bazel compile (gcc)"
vars:
@ -285,8 +280,6 @@ tasks:
- name: version_expansions_gen
variant: generate-tasks-for-version
commands:
# TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
- func: "fetch bazel"
- func: "scons compile"
vars:
targets: >-
@ -310,8 +303,6 @@ tasks:
- name: version_expansions_gen
variant: generate-tasks-for-version
commands:
# TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
- func: "fetch bazel"
- func: "scons compile"
vars:
targets: >-
@ -336,8 +327,6 @@ tasks:
- name: version_expansions_gen
variant: generate-tasks-for-version
commands:
# TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
- func: "fetch bazel"
- func: "scons compile"
vars:
targets: >-
@ -361,8 +350,6 @@ tasks:
- name: version_expansions_gen
variant: generate-tasks-for-version
commands:
# TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
- func: "fetch bazel"
- func: "scons compile"
vars:
targets: >-
@ -388,7 +375,6 @@ tasks:
- name: version_expansions_gen
variant: generate-tasks-for-version
commands:
# TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
- func: "fetch bazel"
- func: "bazel run"
vars:
@ -403,7 +389,6 @@ tasks:
- name: version_expansions_gen
variant: generate-tasks-for-version
commands:
# TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
- func: "fetch bazel"
- func: "bazel run"
vars:

3
evergreen/get_bazelisk.sh
View File

@ -33,8 +33,7 @@ if [[ $ARCH == "ppc64le" ]]; then
REMOTE_PATH=https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-${ARCH}
LOCAL_PATH=$TMPDIR/bazel
else
# TODO(SERVER-81038): remove once bazel/bazelisk is self-hosted.
REMOTE_PATH=https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-${OS}-${ARCH}${EXT}
REMOTE_PATH=https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-${OS}-${ARCH}${EXT}
LOCAL_PATH=$TMPDIR/bazelisk
fi

27
site_scons/site_tools/integrate_bazel.py
View File

@ -15,6 +15,7 @@ import time
from typing import List, Dict, Set, Tuple, Any
import urllib.request
import requests
from retry import retry
import sys
import socket
@ -47,6 +48,16 @@ _SANITIZER_MAP = {
_S3_HASH_MAPPING = {
"https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-ppc64le":
"dd21c75817533ff601bf797e64f0eb2f7f6b813af26c829f0bda30e328caef46",
"https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-darwin-amd64":
"f2ba5f721a995b54bab68c6b76a340719888aa740310e634771086b6d1528ecd",
"https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-darwin-arm64":
"69fa21cd2ccffc2f0970c21aa3615484ba89e3553ecce1233a9d8ad9570d170e",
"https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-linux-amd64":
"d28b588ac0916abd6bf02defb5433f6eddf7cba35ffa808eabb65a44aab226f7",
"https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-linux-arm64":
"861a16ba9979613e70bd3d2f9d9ab5e3b59fe79471c5753acdc9c431ab6c9d94",
"https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-windows-amd64.exe":
"d04555245a99dfb628e33da24e2b9198beb8f46d7e7661c313eb045f6a59f5e4",
}
@ -486,6 +497,11 @@ def generate_bazel_info_for_ninja(env: SCons.Environment.Environment) -> None:
env["NINJA_BAZEL_INPUTS"] = ninja_bazel_ins
@retry(tries=5, delay=3)
def download_path_with_retry(*args, **kwargs):
urllib.request.urlretrieve(*args, **kwargs)
def sha256_file(filename: str) -> str:
sha256_hash = hashlib.sha256()
with open(filename, "rb") as f:
@ -544,20 +560,19 @@ def generate(env: SCons.Environment.Environment) -> None:
bazel_bin_dir, "bazel") if normalized_arch in ["ppc64le"] else os.path.join(
bazel_bin_dir, "bazelisk")
# TODO(SERVER-81038): remove once bazel/bazelisk is self-hosted.
if not os.path.exists(bazel_executable):
print(f"Downloading {bazel_executable}...")
# TODO(SERVER-86050): remove the branch once bazelisk is built on s390x & ppc64le
if normalized_arch in ["ppc64le"]:
s3_path = f"https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-{normalized_arch}"
urllib.request.urlretrieve(s3_path, bazel_executable)
verify_s3_hash(s3_path, bazel_executable)
else:
ext = ".exe" if normalized_os == "windows" else ""
os_str = normalized_os.replace("macos", "darwin")
urllib.request.urlretrieve(
f"https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-{os_str}-{normalized_arch}{ext}",
bazel_executable)
s3_path = f"https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-{os_str}-{normalized_arch}{ext}"
download_path_with_retry(s3_path, bazel_executable)
verify_s3_hash(s3_path, bazel_executable)
print(f"Downloaded {bazel_executable}")
os.chmod(bazel_executable, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
else: