SERVER-103540 Use a common, cross-platform --workspace_status_command (#44301)

GitOrigin-RevId: 643fd10abadc2ef90f03fe5a0aaee2f6ce142207

.bazelrc

@@ -588,8 +588,8 @@ try-import %workspace%/.bazelrc.evergreen
 # local default dev settings
 try-import %workspace%/.bazelrc.common_bes
 
-# local default dev settings
-try-import %workspace%/.bazelrc.mongo_variables
+# Settings unconditionally written by the wrapper hook
+try-import %workspace%/.bazelrc.wrapper_hook
 
 # local git version info
 try-import %workspace%/.bazelrc.git

.gitignore

@@ -299,6 +299,7 @@ buildozer
 .bazelrc.mongo_version
 .bazelrc.compiledb
 .bazelrc.sync
+.bazelrc.wrapper_hook
 .compiledb
 .bazelrc.xcode
 .bazelrc.bazelisk

bazel/resmoke/workspace_status.sh

@@ -1,22 +0,0 @@
-#! /bin/sh
-
-# This script is used as a workspace status command
-#    bazel test --workspace_status_command=bazel/resmoke/workspace_status.sh
-# to populate key-value pairs in bazel-out/volatile-status.txt and stable-status.txt.
-# These files and the key-values can be consumed by bazel rules, but bazel
-# pretends volatile-status never changes when deciding what to rebuild.
-
-# Evergreen expansions used primarily for Resmoke telemetry
-echo build_id ${build_id}
-echo distro_id ${distro_id}
-echo execution ${execution}
-echo project ${project}
-echo revision ${revision}
-echo revision_order_id ${revision_order_id}
-echo task_id ${task_id}
-echo task_name ${task_name}
-echo build_variant ${build_variant}
-echo version_id ${version_id}
-echo requester ${requester}
-
-python bazel/resmoke/workspace_status.py

bazel/workspace_status.py

@@ -1,16 +1,23 @@
+# This script is used as a workspace status command
+#    bazel build --workspace_status_command="python bazel/workspace_status.py" ...
+# to populate key-value pairs in bazel-out/volatile-status.txt and stable-status.txt.
+# These files and the key-values can be consumed by bazel rules, but bazel
+# pretends volatile-status never changes when deciding what to rebuild.
+#
 # Note on imports and python versions:
-#   This file is executed by Bazel using the system Python interpreter. Expect
-#   no packages to be available except those in the standard library. Try to
-#   avoid Python version specific code.
+#   This file is executed by Bazel prior to the build. Expect no packages to
+#   be available except those installed by wrapper_hook.py
 import json
 import os
+import pathlib
 import re
 import sys
 import urllib.request
 
-sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), "../../..")))
-from buildscripts.idl import lib as idl_lib
-from buildscripts.idl.idl import binder as idl_binder
+# Prepend repository root to path, to avoid shadowing from bazel runfiles.
+REPO_ROOT = str(pathlib.Path(__file__).parent.parent)
+sys.path.insert(0, REPO_ROOT)
+from bazel.wrapper_hook.install_modules import setup_python_path
 
 
 def print_stable_key(key: str, value: str):
@@ -24,6 +31,15 @@ def print_volatile_key(key: str, value: str):
 
 
 def print_feature_flag_status():
+    setup_python_path()
+    try:
+        from buildscripts.idl import lib as idl_lib
+        from buildscripts.idl.idl import binder as idl_binder
+    except ModuleNotFoundError:
+        # Modules will not be installed by wrapper_hook.py during the
+        # builds it performs internally (lint, compiledb). In these
+        # situations, these volatile keys can simply be skipped.
+        return
     all_flags = idl_lib.get_all_feature_flags()
 
     enabled_flags = [
@@ -74,7 +90,27 @@ def print_test_runtimes_status():
     print_volatile_key("TEST_RUNTIMES", json.dumps(stats))
 
 
+def print_evergreen_expansions():
+    for expansion in [
+        "build_id",
+        "distro_id",
+        "execution",
+        "project",
+        "revision",
+        "revision_order_id",
+        "task_id",
+        "task_name",
+        "build_variant",
+        "version_id",
+        "requester",
+    ]:
+        value = os.environ.get(expansion, "")
+        if value:
+            print_volatile_key(expansion, value)
+
+
 def main():
+    print_evergreen_expansions()
     print_feature_flag_status()
     print_test_runtimes_status()
 

bazel/wrapper_hook/BUILD.bazel

@@ -12,10 +12,10 @@ py_library(
         "install_modules.py",
         "lint.py",
         "plus_interface.py",
-        "set_mongo_variables.py",
         "wrapper_debug.py",
         "wrapper_hook.py",
         "wrapper_util.py",
+        "write_wrapper_hook_bazelrc.py",
     ],
     visibility = ["//visibility:public"],
 )

bazel/wrapper_hook/compiledb.py

@@ -11,7 +11,7 @@ import sys
 REPO_ROOT = pathlib.Path(__file__).parent.parent.parent
 sys.path.append(str(REPO_ROOT))
 
-from bazel.wrapper_hook.set_mongo_variables import write_mongo_variables_bazelrc
+from bazel.wrapper_hook.write_wrapper_hook_bazelrc import write_wrapper_hook_bazelrc
 
 
 def run_pty_command(cmd):
@@ -46,7 +46,7 @@ def run_pty_command(cmd):
 
 def generate_compiledb(bazel_bin, persistent_compdb, enterprise):
     # compiledb ignores command line args so just make a version rc file in anycase
-    write_mongo_variables_bazelrc([])
+    write_wrapper_hook_bazelrc([])
     if persistent_compdb:
         info_proc = subprocess.run(
             [bazel_bin, "info", "output_base"], capture_output=True, text=True

bazel/wrapper_hook/install_modules.py

@@ -111,7 +111,7 @@ def install_modules(bazel, args):
         with open(lockfile_hash_file, "w") as f:
             f.write(current_hash)
 
-    deps = ["retry", "gitpython", "requests", "timeout-decorator", "boto3"]
+    deps = ["retry", "gitpython", "requests", "timeout-decorator", "boto3", "pyyaml", "pymongo"]
     deps_installed = []
     deps_needed = search_for_modules(
         deps, deps_installed, lockfile_changed=old_hash != current_hash
@@ -139,6 +139,7 @@ def install_modules(bazel, args):
                 "--remote_download_all",
                 "--bes_backend=",
                 "--bes_results_url=",
+                "--workspace_status_command=",
             ]
         )
         if proc.returncode != 0:

bazel/wrapper_hook/wrapper_hook.py

@@ -92,7 +92,7 @@ def main():
     # from bazel.wrapper_hook.git_age_check import check as git_age_check
     from bazel.wrapper_hook.lint import LinterFail
     from bazel.wrapper_hook.plus_interface import check_bazel_command_type, test_runner_interface
-    from bazel.wrapper_hook.set_mongo_variables import write_mongo_variables_bazelrc
+    from bazel.wrapper_hook.write_wrapper_hook_bazelrc import write_wrapper_hook_bazelrc
 
     th_all_header, hdr_state_all_header = spawn_all_headers_thread(REPO_ROOT)
 
@@ -137,7 +137,7 @@ def main():
 
         engflow_auth(args)
         write_workstation_bazelrc(args)
-        write_mongo_variables_bazelrc(args)
+        write_wrapper_hook_bazelrc(args)
         # Disable git age check for now, to avoid issues wth merge commits
         # git_age_check()
         check_resource()

bazel/wrapper_hook/write_wrapper_hook_bazelrc.py

@@ -3,6 +3,7 @@ import os
 import pathlib
 import platform
 import subprocess
+import sys
 
 ARCH_NORMALIZE_MAP = {
     "amd64": "x86_64",
@@ -27,12 +28,21 @@ def get_mongo_version(args):
     return proc.stdout.strip()[1:]
 
 
-def write_mongo_variables_bazelrc(args):
+def write_wrapper_hook_bazelrc(args):
     mongo_version = get_mongo_version(args)
     mongo_arch = get_mongo_arch(args)
 
+    python = sys.executable
+    workspace_status = os.path.join("bazel", "workspace_status.py")
+    if sys.platform == "win32":
+        # Bazel processes the workspace_status_command, breaking Windows
+        # paths. Add escaping so that the bazelrc contents is like:
+        # --workspace_status_command="Z:\\tmp\\python.exe bazel\\workspace_status.py"
+        python = python.replace("\\", "\\\\")
+        workspace_status = workspace_status.replace("\\", "\\\\")
+
     repo_root = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent
-    version_file = os.path.join(repo_root, ".bazelrc.mongo_variables")
+    version_file = os.path.join(repo_root, ".bazelrc.wrapper_hook")
     existing_hash = ""
     if os.path.exists(version_file):
         with open(version_file, encoding="utf-8") as f:
@@ -41,7 +51,10 @@ def write_mongo_variables_bazelrc(args):
     bazelrc_contents = f"""
 common --define=MONGO_ARCH={mongo_arch}
 common --define=MONGO_VERSION={mongo_version}
+
+build --workspace_status_command="{python} {workspace_status}"
 """
+
     current_hash = hashlib.md5(bazelrc_contents.encode()).hexdigest()
     if existing_hash != current_hash:
         with open(version_file, "w", encoding="utf-8") as f:

etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml

@@ -486,7 +486,6 @@ tasks:
           suite: //buildscripts/resmokeconfig:core
           compiling_for_test: true
           bazel_args: >-
-            --workspace_status_command="bazel/resmoke/workspace_status.sh"
             --test_sharding_strategy=forced=16
             --test_arg=--testTimeout=960
             --test_timeout=1500
@@ -507,8 +506,6 @@ tasks:
           jstestfuzz_vars: --jsTestsDir ../jstests
           suite: //buildscripts/resmokeconfig:jstestfuzz
           npm_command: jstestfuzz
-          bazel_args: >-
-            --workspace_status_command="bazel/resmoke/workspace_status.sh"
 
   # Experimental task running the multiversion sanity check bazel targets. To be removed with SERVER-103537.
   - <<: *gen_task_template
@@ -522,9 +519,6 @@ tasks:
       ]
     commands:
       - func: "generate resmoke tasks"
-        vars:
-          bazel_args: >-
-            --workspace_status_command="bazel/resmoke/workspace_status.sh"
       - func: "initialize multiversion tasks"
         vars:
           //buildscripts/resmokeconfig:bazel_multiversion_sanity_check_last_continuous_new_new_old: last_continuous

tools/bazel

@@ -200,7 +200,6 @@ fi
 # find existing python installs
 cur_dir=$(basename $REPO_ROOT)
 python=""
-standard_python_loc="$(readlink "$REPO_ROOT/bazel-$cur_dir")/../../external/_main~setup_mongo_python_toolchains~py_${os}_${ARCH}/dist/bin/python3"
 if [ -d $REPO_ROOT/bazel-$cur_dir ]; then
   python="$(readlink "$REPO_ROOT/bazel-$cur_dir")/../../external/_main~setup_mongo_python_toolchains~py_${os}_${ARCH}/dist/bin/python3"
 else