From f6e67fbdaaca6fb0c83aa77628687d7295aabe1f Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 2 May 2025 16:04:36 -0500 Subject: [PATCH] SERVER-104622 use engflow's remote asset api (#35669) GitOrigin-RevId: e2c17b1fb8a8657d6140684132be0b8cacfc2ab1 --- .bazelrc | 5 +++++ bazel/wrapper_hook/install_modules.py | 9 ++++++++- buildscripts/mongo_toolchain.py | 4 +++- buildscripts/unittest_grouper.py | 2 -- etc/evergreen_yml_components/tasks/misc_tasks.yml | 2 ++ tools/bazel | 4 ++-- tools/bazel.bat | 3 ++- 7 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.bazelrc b/.bazelrc index fa4e91a7c8c..028ff2f687b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -50,6 +50,9 @@ common --cache_test_results=no # Stream the test output so users can see the test progression. common --test_output=streamed +# when using remote asset api, fallback to the direct urls to download +common --experimental_remote_downloader_local_fallback + # Build only what is necessary to run the *_test and test_suite rules that were # not filtered due to their size, timeout, tag, or language. test --build_tests_only @@ -393,6 +396,7 @@ common:windows --remote_executor= common:macos --remote_executor= # Enable remote cache (also necessary for remote build execution): +common --experimental_remote_downloader=grpcs://sodalite.cluster.engflow.com common --remote_cache=grpcs://sodalite.cluster.engflow.com common --bes_backend=grpcs://sodalite.cluster.engflow.com common --bes_results_url=https://sodalite.cluster.engflow.com/invocation/ @@ -429,6 +433,7 @@ common:mod-scanner --remote_download_regex=.*\.mod_scanner_decls.json$ # by passing "--config=local" on the bazel command line --config=local common:local --remote_executor= +common:local --experimental_remote_downloader= common:local --remote_cache= common:local --bes_backend= common:local --bes_results_url= diff --git a/bazel/wrapper_hook/install_modules.py b/bazel/wrapper_hook/install_modules.py index c5dbed46849..febac1e3901 100644 --- a/bazel/wrapper_hook/install_modules.py +++ b/bazel/wrapper_hook/install_modules.py @@ -120,7 +120,14 @@ def install_modules(bazel): if need_to_install: subprocess.run( - [bazel, "build", "--config=local"] + ["@poetry//:library_" + dep for dep in deps_needed] + [ + bazel, + "build", + "--remote_download_all", + "--bes_backend=", + "--bes_results_url=", + ] + + ["@poetry//:library_" + dep for dep in deps_needed] ) deps_missing = search_for_modules(deps_needed, deps_installed) if deps_missing: diff --git a/buildscripts/mongo_toolchain.py b/buildscripts/mongo_toolchain.py index 541bde8dba9..4079988379e 100644 --- a/buildscripts/mongo_toolchain.py +++ b/buildscripts/mongo_toolchain.py @@ -102,7 +102,9 @@ def _execute_bazel(argv): def _fetch_bazel_toolchain(version: str) -> None: try: - _execute_bazel(["build", "--config=local", f"@mongo_toolchain_{version}//:all"]) + _execute_bazel( + ["build", "--bes_backend=", "--bes_results_url=", f"@mongo_toolchain_{version}//:all"] + ) except subprocess.CalledProcessError as e: raise MongoToolchainNotFoundError( f"Failed to fetch bazel toolchain: `{e.cmd}` exited with code {e.returncode}" diff --git a/buildscripts/unittest_grouper.py b/buildscripts/unittest_grouper.py index 023f4b7cfd0..f1db2064f61 100644 --- a/buildscripts/unittest_grouper.py +++ b/buildscripts/unittest_grouper.py @@ -155,8 +155,6 @@ def validate_bazel_groups(generate_report, fix): "--implicit_deps=False", "--tool_deps=False", "--include_aspects=False", - "--remote_executor=", - "--remote_cache=", "--bes_backend=", "--bes_results_url=", ] diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index d0c24e57bf3..d32d6912b1e 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -696,6 +696,7 @@ tasks: - func: "cleanup environment" - func: "set up venv" - func: "upload pip requirements" + - func: "get engflow creds" - command: subprocess.exec type: test params: @@ -903,6 +904,7 @@ tasks: - func: "cleanup environment" - func: "set up venv" - func: "upload pip requirements" + - func: "get engflow creds" - command: subprocess.exec params: binary: bash diff --git a/tools/bazel b/tools/bazel index 81bb9e9cd27..b76a496c6bd 100755 --- a/tools/bazel +++ b/tools/bazel @@ -137,7 +137,7 @@ fi if [ "$skip_python" == "0" ]; then # known list of commands to skip - skip_commands=("clean" "version" "shutdown") + skip_commands=("clean" "version" "shutdown" "info") for command in "${skip_commands[@]}" do @@ -167,7 +167,7 @@ fi # if no python use bazel to install one if [[ "$python" = "" ]] || [ ! -f $python ]; then >&2 echo "python prereq missing, using bazel to install python..." - >&2 $bazel_real build --config=local @py_${os}_${ARCH}//:all + >&2 $bazel_real build --bes_backend= --bes_results_url= @py_${os}_${ARCH}//:all if [[ $? != 0 ]]; then if [[ ! -z "$CI" ]] || [[ $MONGO_BAZEL_WRAPPER_FALLBACK == 1 ]]; then >&2 echo "wrapper script failed to install python! falling back to normal bazel call..." diff --git a/tools/bazel.bat b/tools/bazel.bat index 0289b561d3f..edc5e1c04ba 100644 --- a/tools/bazel.bat +++ b/tools/bazel.bat @@ -36,6 +36,7 @@ if !current_bazel_command!=="" set skip_python="1" if !skip_python!=="0" if !current_bazel_command!=="clean" set skip_python="1" if !skip_python!=="0" if !current_bazel_command!=="version" set skip_python="1" if !skip_python!=="0" if !current_bazel_command!=="shutdown" set skip_python="1" +if !skip_python!=="0" if !current_bazel_command!=="info" set skip_python="1" if !skip_python!=="1" ( "%BAZEL_REAL%" %* @@ -48,7 +49,7 @@ if exist %REPO_ROOT%\bazel-%cur_dir% ( ) if not exist "!python!" ( echo python prereq missing, using bazel to install python... 1>&2 - "%BAZEL_REAL%" build --config=local @py_windows_x86_64//:all 1>&2 + "%BAZEL_REAL%" build --bes_backend= --bes_results_url= @py_windows_x86_64//:all 1>&2 if %ERRORLEVEL% NEQ 0 ( if "%CI%"=="" if "%MONGO_BAZEL_WRAPPER_FALLBACK%"=="" exit %ERRORLEVEL%