From 2a588d904af1a472a327e2f0ee2f0796bac923be Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Tue, 25 Nov 2025 09:45:07 -0500 Subject: [PATCH] SERVER-103540 Use a common, cross-platform --workspace_status_command (#44301) GitOrigin-RevId: 643fd10abadc2ef90f03fe5a0aaee2f6ce142207 --- .bazelrc | 4 +- .gitignore | 1 + bazel/resmoke/workspace_status.sh | 22 --------- bazel/{resmoke => }/workspace_status.py | 48 ++++++++++++++++--- bazel/wrapper_hook/BUILD.bazel | 2 +- bazel/wrapper_hook/compiledb.py | 4 +- bazel/wrapper_hook/install_modules.py | 3 +- bazel/wrapper_hook/wrapper_hook.py | 4 +- ...ables.py => write_wrapper_hook_bazelrc.py} | 17 ++++++- .../tasks/resmoke/non_server_teams/tasks.yml | 6 --- tools/bazel | 1 - 11 files changed, 67 insertions(+), 45 deletions(-) delete mode 100755 bazel/resmoke/workspace_status.sh rename bazel/{resmoke => }/workspace_status.py (57%) rename bazel/wrapper_hook/{set_mongo_variables.py => write_wrapper_hook_bazelrc.py} (65%) diff --git a/.bazelrc b/.bazelrc index 9d6618db02e..9425644308e 100644 --- a/.bazelrc +++ b/.bazelrc @@ -588,8 +588,8 @@ try-import %workspace%/.bazelrc.evergreen # local default dev settings try-import %workspace%/.bazelrc.common_bes -# local default dev settings -try-import %workspace%/.bazelrc.mongo_variables +# Settings unconditionally written by the wrapper hook +try-import %workspace%/.bazelrc.wrapper_hook # local git version info try-import %workspace%/.bazelrc.git diff --git a/.gitignore b/.gitignore index 7d3d7834d8a..91f8911c024 100644 --- a/.gitignore +++ b/.gitignore @@ -299,6 +299,7 @@ buildozer .bazelrc.mongo_version .bazelrc.compiledb .bazelrc.sync +.bazelrc.wrapper_hook .compiledb .bazelrc.xcode .bazelrc.bazelisk diff --git a/bazel/resmoke/workspace_status.sh b/bazel/resmoke/workspace_status.sh deleted file mode 100755 index 582177c3dea..00000000000 --- a/bazel/resmoke/workspace_status.sh +++ /dev/null @@ -1,22 +0,0 @@ -#! /bin/sh - -# This script is used as a workspace status command -# bazel test --workspace_status_command=bazel/resmoke/workspace_status.sh -# to populate key-value pairs in bazel-out/volatile-status.txt and stable-status.txt. -# These files and the key-values can be consumed by bazel rules, but bazel -# pretends volatile-status never changes when deciding what to rebuild. - -# Evergreen expansions used primarily for Resmoke telemetry -echo build_id ${build_id} -echo distro_id ${distro_id} -echo execution ${execution} -echo project ${project} -echo revision ${revision} -echo revision_order_id ${revision_order_id} -echo task_id ${task_id} -echo task_name ${task_name} -echo build_variant ${build_variant} -echo version_id ${version_id} -echo requester ${requester} - -python bazel/resmoke/workspace_status.py diff --git a/bazel/resmoke/workspace_status.py b/bazel/workspace_status.py similarity index 57% rename from bazel/resmoke/workspace_status.py rename to bazel/workspace_status.py index 7577a4852ad..4ee67a92a9f 100644 --- a/bazel/resmoke/workspace_status.py +++ b/bazel/workspace_status.py @@ -1,16 +1,23 @@ +# This script is used as a workspace status command +# bazel build --workspace_status_command="python bazel/workspace_status.py" ... +# to populate key-value pairs in bazel-out/volatile-status.txt and stable-status.txt. +# These files and the key-values can be consumed by bazel rules, but bazel +# pretends volatile-status never changes when deciding what to rebuild. +# # Note on imports and python versions: -# This file is executed by Bazel using the system Python interpreter. Expect -# no packages to be available except those in the standard library. Try to -# avoid Python version specific code. +# This file is executed by Bazel prior to the build. Expect no packages to +# be available except those installed by wrapper_hook.py import json import os +import pathlib import re import sys import urllib.request -sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), "../../.."))) -from buildscripts.idl import lib as idl_lib -from buildscripts.idl.idl import binder as idl_binder +# Prepend repository root to path, to avoid shadowing from bazel runfiles. +REPO_ROOT = str(pathlib.Path(__file__).parent.parent) +sys.path.insert(0, REPO_ROOT) +from bazel.wrapper_hook.install_modules import setup_python_path def print_stable_key(key: str, value: str): @@ -24,6 +31,15 @@ def print_volatile_key(key: str, value: str): def print_feature_flag_status(): + setup_python_path() + try: + from buildscripts.idl import lib as idl_lib + from buildscripts.idl.idl import binder as idl_binder + except ModuleNotFoundError: + # Modules will not be installed by wrapper_hook.py during the + # builds it performs internally (lint, compiledb). In these + # situations, these volatile keys can simply be skipped. + return all_flags = idl_lib.get_all_feature_flags() enabled_flags = [ @@ -74,7 +90,27 @@ def print_test_runtimes_status(): print_volatile_key("TEST_RUNTIMES", json.dumps(stats)) +def print_evergreen_expansions(): + for expansion in [ + "build_id", + "distro_id", + "execution", + "project", + "revision", + "revision_order_id", + "task_id", + "task_name", + "build_variant", + "version_id", + "requester", + ]: + value = os.environ.get(expansion, "") + if value: + print_volatile_key(expansion, value) + + def main(): + print_evergreen_expansions() print_feature_flag_status() print_test_runtimes_status() diff --git a/bazel/wrapper_hook/BUILD.bazel b/bazel/wrapper_hook/BUILD.bazel index dea64c09a7e..77727357004 100644 --- a/bazel/wrapper_hook/BUILD.bazel +++ b/bazel/wrapper_hook/BUILD.bazel @@ -12,10 +12,10 @@ py_library( "install_modules.py", "lint.py", "plus_interface.py", - "set_mongo_variables.py", "wrapper_debug.py", "wrapper_hook.py", "wrapper_util.py", + "write_wrapper_hook_bazelrc.py", ], visibility = ["//visibility:public"], ) diff --git a/bazel/wrapper_hook/compiledb.py b/bazel/wrapper_hook/compiledb.py index a430cb94076..cffc4173ab6 100644 --- a/bazel/wrapper_hook/compiledb.py +++ b/bazel/wrapper_hook/compiledb.py @@ -11,7 +11,7 @@ import sys REPO_ROOT = pathlib.Path(__file__).parent.parent.parent sys.path.append(str(REPO_ROOT)) -from bazel.wrapper_hook.set_mongo_variables import write_mongo_variables_bazelrc +from bazel.wrapper_hook.write_wrapper_hook_bazelrc import write_wrapper_hook_bazelrc def run_pty_command(cmd): @@ -46,7 +46,7 @@ def run_pty_command(cmd): def generate_compiledb(bazel_bin, persistent_compdb, enterprise): # compiledb ignores command line args so just make a version rc file in anycase - write_mongo_variables_bazelrc([]) + write_wrapper_hook_bazelrc([]) if persistent_compdb: info_proc = subprocess.run( [bazel_bin, "info", "output_base"], capture_output=True, text=True diff --git a/bazel/wrapper_hook/install_modules.py b/bazel/wrapper_hook/install_modules.py index 91768e46a8f..34d23b75025 100644 --- a/bazel/wrapper_hook/install_modules.py +++ b/bazel/wrapper_hook/install_modules.py @@ -111,7 +111,7 @@ def install_modules(bazel, args): with open(lockfile_hash_file, "w") as f: f.write(current_hash) - deps = ["retry", "gitpython", "requests", "timeout-decorator", "boto3"] + deps = ["retry", "gitpython", "requests", "timeout-decorator", "boto3", "pyyaml", "pymongo"] deps_installed = [] deps_needed = search_for_modules( deps, deps_installed, lockfile_changed=old_hash != current_hash @@ -139,6 +139,7 @@ def install_modules(bazel, args): "--remote_download_all", "--bes_backend=", "--bes_results_url=", + "--workspace_status_command=", ] ) if proc.returncode != 0: diff --git a/bazel/wrapper_hook/wrapper_hook.py b/bazel/wrapper_hook/wrapper_hook.py index 3822ccfcab1..2ae037141d7 100644 --- a/bazel/wrapper_hook/wrapper_hook.py +++ b/bazel/wrapper_hook/wrapper_hook.py @@ -92,7 +92,7 @@ def main(): # from bazel.wrapper_hook.git_age_check import check as git_age_check from bazel.wrapper_hook.lint import LinterFail from bazel.wrapper_hook.plus_interface import check_bazel_command_type, test_runner_interface - from bazel.wrapper_hook.set_mongo_variables import write_mongo_variables_bazelrc + from bazel.wrapper_hook.write_wrapper_hook_bazelrc import write_wrapper_hook_bazelrc th_all_header, hdr_state_all_header = spawn_all_headers_thread(REPO_ROOT) @@ -137,7 +137,7 @@ def main(): engflow_auth(args) write_workstation_bazelrc(args) - write_mongo_variables_bazelrc(args) + write_wrapper_hook_bazelrc(args) # Disable git age check for now, to avoid issues wth merge commits # git_age_check() check_resource() diff --git a/bazel/wrapper_hook/set_mongo_variables.py b/bazel/wrapper_hook/write_wrapper_hook_bazelrc.py similarity index 65% rename from bazel/wrapper_hook/set_mongo_variables.py rename to bazel/wrapper_hook/write_wrapper_hook_bazelrc.py index 820ea7c06f1..e0f156e5e9c 100644 --- a/bazel/wrapper_hook/set_mongo_variables.py +++ b/bazel/wrapper_hook/write_wrapper_hook_bazelrc.py @@ -3,6 +3,7 @@ import os import pathlib import platform import subprocess +import sys ARCH_NORMALIZE_MAP = { "amd64": "x86_64", @@ -27,12 +28,21 @@ def get_mongo_version(args): return proc.stdout.strip()[1:] -def write_mongo_variables_bazelrc(args): +def write_wrapper_hook_bazelrc(args): mongo_version = get_mongo_version(args) mongo_arch = get_mongo_arch(args) + python = sys.executable + workspace_status = os.path.join("bazel", "workspace_status.py") + if sys.platform == "win32": + # Bazel processes the workspace_status_command, breaking Windows + # paths. Add escaping so that the bazelrc contents is like: + # --workspace_status_command="Z:\\tmp\\python.exe bazel\\workspace_status.py" + python = python.replace("\\", "\\\\") + workspace_status = workspace_status.replace("\\", "\\\\") + repo_root = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent - version_file = os.path.join(repo_root, ".bazelrc.mongo_variables") + version_file = os.path.join(repo_root, ".bazelrc.wrapper_hook") existing_hash = "" if os.path.exists(version_file): with open(version_file, encoding="utf-8") as f: @@ -41,7 +51,10 @@ def write_mongo_variables_bazelrc(args): bazelrc_contents = f""" common --define=MONGO_ARCH={mongo_arch} common --define=MONGO_VERSION={mongo_version} + +build --workspace_status_command="{python} {workspace_status}" """ + current_hash = hashlib.md5(bazelrc_contents.encode()).hexdigest() if existing_hash != current_hash: with open(version_file, "w", encoding="utf-8") as f: diff --git a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml index f22ae795afd..e8d56578059 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml @@ -486,7 +486,6 @@ tasks: suite: //buildscripts/resmokeconfig:core compiling_for_test: true bazel_args: >- - --workspace_status_command="bazel/resmoke/workspace_status.sh" --test_sharding_strategy=forced=16 --test_arg=--testTimeout=960 --test_timeout=1500 @@ -507,8 +506,6 @@ tasks: jstestfuzz_vars: --jsTestsDir ../jstests suite: //buildscripts/resmokeconfig:jstestfuzz npm_command: jstestfuzz - bazel_args: >- - --workspace_status_command="bazel/resmoke/workspace_status.sh" # Experimental task running the multiversion sanity check bazel targets. To be removed with SERVER-103537. - <<: *gen_task_template @@ -522,9 +519,6 @@ tasks: ] commands: - func: "generate resmoke tasks" - vars: - bazel_args: >- - --workspace_status_command="bazel/resmoke/workspace_status.sh" - func: "initialize multiversion tasks" vars: //buildscripts/resmokeconfig:bazel_multiversion_sanity_check_last_continuous_new_new_old: last_continuous diff --git a/tools/bazel b/tools/bazel index 5b65ceff524..8e77dfe68d5 100755 --- a/tools/bazel +++ b/tools/bazel @@ -200,7 +200,6 @@ fi # find existing python installs cur_dir=$(basename $REPO_ROOT) python="" -standard_python_loc="$(readlink "$REPO_ROOT/bazel-$cur_dir")/../../external/_main~setup_mongo_python_toolchains~py_${os}_${ARCH}/dist/bin/python3" if [ -d $REPO_ROOT/bazel-$cur_dir ]; then python="$(readlink "$REPO_ROOT/bazel-$cur_dir")/../../external/_main~setup_mongo_python_toolchains~py_${os}_${ARCH}/dist/bin/python3" else