SERVER-104622 use engflow's remote asset api (#35669)

GitOrigin-RevId: e2c17b1fb8a8657d6140684132be0b8cacfc2ab1

.bazelrc

@@ -50,6 +50,9 @@ common --cache_test_results=no
 # Stream the test output so users can see the test progression.
 common --test_output=streamed
 
+# when using remote asset api, fallback to the direct urls to download
+common --experimental_remote_downloader_local_fallback
+
 # Build only what is necessary to run the *_test and test_suite rules that were
 # not filtered due to their size, timeout, tag, or language.
 test --build_tests_only
@@ -393,6 +396,7 @@ common:windows --remote_executor=
 common:macos --remote_executor=
 
 # Enable remote cache (also necessary for remote build execution):
+common --experimental_remote_downloader=grpcs://sodalite.cluster.engflow.com
 common --remote_cache=grpcs://sodalite.cluster.engflow.com
 common --bes_backend=grpcs://sodalite.cluster.engflow.com
 common --bes_results_url=https://sodalite.cluster.engflow.com/invocation/
@@ -429,6 +433,7 @@ common:mod-scanner --remote_download_regex=.*\.mod_scanner_decls.json$
 # by passing "--config=local" on the bazel command line
 --config=local
 common:local --remote_executor=
+common:local --experimental_remote_downloader=
 common:local --remote_cache=
 common:local --bes_backend=
 common:local --bes_results_url=

bazel/wrapper_hook/install_modules.py

@@ -120,7 +120,14 @@ def install_modules(bazel):
 
     if need_to_install:
         subprocess.run(
-            [bazel, "build", "--config=local"] + ["@poetry//:library_" + dep for dep in deps_needed]
+            [
+                bazel,
+                "build",
+                "--remote_download_all",
+                "--bes_backend=",
+                "--bes_results_url=",
+            ]
+            + ["@poetry//:library_" + dep for dep in deps_needed]
         )
         deps_missing = search_for_modules(deps_needed, deps_installed)
         if deps_missing:

buildscripts/mongo_toolchain.py

@@ -102,7 +102,9 @@ def _execute_bazel(argv):
 
 def _fetch_bazel_toolchain(version: str) -> None:
     try:
-        _execute_bazel(["build", "--config=local", f"@mongo_toolchain_{version}//:all"])
+        _execute_bazel(
+            ["build", "--bes_backend=", "--bes_results_url=", f"@mongo_toolchain_{version}//:all"]
+        )
     except subprocess.CalledProcessError as e:
         raise MongoToolchainNotFoundError(
             f"Failed to fetch bazel toolchain: `{e.cmd}` exited with code {e.returncode}"

buildscripts/unittest_grouper.py

@@ -155,8 +155,6 @@ def validate_bazel_groups(generate_report, fix):
         "--implicit_deps=False",
         "--tool_deps=False",
         "--include_aspects=False",
-        "--remote_executor=",
-        "--remote_cache=",
         "--bes_backend=",
         "--bes_results_url=",
     ]

etc/evergreen_yml_components/tasks/misc_tasks.yml

@@ -696,6 +696,7 @@ tasks:
       - func: "cleanup environment"
       - func: "set up venv"
       - func: "upload pip requirements"
+      - func: "get engflow creds"
       - command: subprocess.exec
         type: test
         params:
@@ -903,6 +904,7 @@ tasks:
       - func: "cleanup environment"
       - func: "set up venv"
       - func: "upload pip requirements"
+      - func: "get engflow creds"
       - command: subprocess.exec
         params:
           binary: bash

tools/bazel

@@ -137,7 +137,7 @@ fi
 
 if [ "$skip_python" == "0" ]; then
   # known list of commands to skip
-  skip_commands=("clean" "version" "shutdown")
+  skip_commands=("clean" "version" "shutdown" "info")
 
   for command in "${skip_commands[@]}"
   do
@@ -167,7 +167,7 @@ fi
 # if no python use bazel to install one
 if [[ "$python" = "" ]] || [ ! -f $python ]; then
   >&2 echo "python prereq missing, using bazel to install python..."
-  >&2 $bazel_real build --config=local @py_${os}_${ARCH}//:all
+  >&2 $bazel_real build --bes_backend= --bes_results_url= @py_${os}_${ARCH}//:all
   if [[ $? != 0 ]]; then
     if [[ ! -z "$CI" ]] || [[ $MONGO_BAZEL_WRAPPER_FALLBACK == 1 ]]; then
       >&2 echo "wrapper script failed to install python! falling back to normal bazel call..."

tools/bazel.bat

@@ -36,6 +36,7 @@ if !current_bazel_command!=="" set skip_python="1"
 if !skip_python!=="0" if !current_bazel_command!=="clean" set skip_python="1"
 if !skip_python!=="0" if !current_bazel_command!=="version" set skip_python="1"
 if !skip_python!=="0" if !current_bazel_command!=="shutdown" set skip_python="1"
+if !skip_python!=="0" if !current_bazel_command!=="info" set skip_python="1"
 
 if !skip_python!=="1" (
     "%BAZEL_REAL%" %*
@@ -48,7 +49,7 @@ if exist %REPO_ROOT%\bazel-%cur_dir% (
 )
 if not exist "!python!" (
     echo python prereq missing, using bazel to install python... 1>&2
-    "%BAZEL_REAL%" build --config=local @py_windows_x86_64//:all 1>&2
+    "%BAZEL_REAL%" build --bes_backend= --bes_results_url= @py_windows_x86_64//:all 1>&2
     
     if %ERRORLEVEL% NEQ 0 (
         if "%CI%"=="" if "%MONGO_BAZEL_WRAPPER_FALLBACK%"=="" exit %ERRORLEVEL%