From 0e3925dfeab962c8ab49b22eea2bbba8ff4ec70b Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Mon, 23 Jun 2025 09:16:18 -0400 Subject: [PATCH] SERVER-106530 Use an exclusive base port for resmoke targets (#37578) GitOrigin-RevId: a63decba77fe630a8618290066a58cf814ccdf44 --- .bazelrc | 6 ++ bazel/resmoke/resmoke.bzl | 10 +++- bazel/resmoke/resmoke_shim.py | 6 ++ buildscripts/BUILD.bazel | 10 ++++ buildscripts/bazel_local_resources.py | 58 +++++++++++++++++++ .../tasks/compile_tasks_shared.yml | 1 + poetry.lock | 10 ++-- pyproject.toml | 1 + 8 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 buildscripts/bazel_local_resources.py diff --git a/.bazelrc b/.bazelrc index 647920b8e61..052d7a19df7 100644 --- a/.bazelrc +++ b/.bazelrc @@ -70,6 +70,12 @@ common --experimental_remote_downloader_local_fallback # not filtered due to their size, timeout, tag, or language. test --build_tests_only +# Port blocks that can be exclusively used by one test action at a time, representing +# a range of 250 ports up until the next block. +# The contract for acquiring is detailed in buildscripts/bazel_local_resources.py. +test --local_resources=port_block=40 # 40 distinct port blocks +test --define=LOCAL_RESOURCES="port_block=20000,20250,20500,20750,21000,21250,21500,21750,22000,22250,22500,22750,23000,23250,23500,23750,24000,24250,24500,24750,25000,25250,25500,25750,26000,26250,26500,26750,27000,27250,27500,27750,28000,28250,28500,28750,29000,29250,29500,29750" + # Pin down the OSS LLVM Clang version for MacOS builds. common:macos --repo_env=LLVM_VERSION=19 diff --git a/bazel/resmoke/resmoke.bzl b/bazel/resmoke/resmoke.bzl index 4490645dde7..34e84e65a3e 100644 --- a/bazel/resmoke/resmoke.bzl +++ b/bazel/resmoke/resmoke.bzl @@ -115,7 +115,10 @@ def resmoke_suite_test( "//bazel/resmoke:in_evergreen_enabled": ["//:installed-dist-test"], "//conditions:default": ["//:install-dist-test"], }), - deps = deps + [resmoke], + deps = deps + [ + resmoke, + "//buildscripts:bazel_local_resources", + ], main = resmoke_shim, args = [ "run", @@ -123,7 +126,10 @@ def resmoke_suite_test( "--multiversionDir=multiversion_binaries", "--continueOnFailure", ] + extra_args + resmoke_args, - tags = tags + ["no-cache", "local"], + tags = tags + ["no-cache", "local", "resources:port_block:1"], timeout = timeout, + env = { + "LOCAL_RESOURCES": "$(LOCAL_RESOURCES)", + }, **kwargs ) diff --git a/bazel/resmoke/resmoke_shim.py b/bazel/resmoke/resmoke_shim.py index 7adeb80fd6b..1fdded59638 100644 --- a/bazel/resmoke/resmoke_shim.py +++ b/bazel/resmoke/resmoke_shim.py @@ -5,6 +5,7 @@ from functools import cache REPO_ROOT = pathlib.Path(__file__).parent.parent.parent sys.path.append(str(REPO_ROOT)) +from buildscripts.bazel_local_resources import acquire_local_resource from buildscripts.resmokelib import cli @@ -78,4 +79,9 @@ if __name__ == "__main__": link_path, ) + lock, base_port = acquire_local_resource("port_block") + resmoke_args.append(f"--basePort={base_port}") + cli.main(resmoke_args) + + lock.release() diff --git a/buildscripts/BUILD.bazel b/buildscripts/BUILD.bazel index fffbce6efc0..49d2cec346b 100644 --- a/buildscripts/BUILD.bazel +++ b/buildscripts/BUILD.bazel @@ -277,6 +277,16 @@ py_binary( ], ) +py_binary( + name = "bazel_local_resources", + srcs = ["bazel_local_resources.py"], + visibility = ["//visibility:public"], + deps = [dependency( + "filelock", + group = "testing", + )], +) + # TODO(SERVER-105817): The following library is autogenerated, please split these out into individual python targets py_library( name = "all_python_files", diff --git a/buildscripts/bazel_local_resources.py b/buildscripts/bazel_local_resources.py new file mode 100644 index 00000000000..64ac35ac246 --- /dev/null +++ b/buildscripts/bazel_local_resources.py @@ -0,0 +1,58 @@ +""" +A utility for acquiring exclusive access to named local resources from within a +locally run bazel action. + +The resource should be defined in the bazel configuration: + # 4 possible port blocks, each representing the ports that can be used up until the next block. + --local_resources=port_block=4 + --define=LOCAL_RESOURCES="port_block=20000,20250,20500,20750" + +The target that needs the resource should add the tag: + tags = ["resources:port_block:1"] # This action needs one port range +And include LOCAL_RESOURCES in the action environment: + env = { + "LOCAL_RESOURCES": "$(LOCAL_RESOURCES)", + } + +The action should use `acquire_local_resource` to get a lock on the local resource: + lock, port_block = acquire_local_resource("port_block") + # Use port_block, no other action will be using the same one. + lock.release() +""" + +import os +import pathlib +import tempfile +from functools import cache + +from filelock import FileLock, Timeout + + +@cache +def _parse_local_resources() -> dict: + resources = {} + for resource in os.environ.get("LOCAL_RESOURCES").split(";"): + name, values = resource.split("=", 1) + resources[name] = values.split(",") + return resources + + +def acquire_local_resource(resource_name: str) -> (FileLock, str): + local_resources = _parse_local_resources() + if resource_name not in local_resources: + raise Exception( + f"Resource {resource_name} not found in LOCAL_RESOURCES. LOCAL_RESOURCES are: {local_resources}" + ) + + lock_dir = pathlib.Path(tempfile.gettempdir(), "bazel_local_resources", resource_name) + lock_dir.mkdir(parents=True, exist_ok=True) + + for resource in local_resources[resource_name]: + try: + lock = FileLock(lock_dir / f"{resource}.lock") + lock.acquire(timeout=0) + return lock, resource + except Timeout: + continue + + raise Exception(f"Could not acquire a lock for resource {resource_name}") diff --git a/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml b/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml index 4efcd09c3ea..2a8ba2860dd 100644 --- a/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml +++ b/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml @@ -206,6 +206,7 @@ tasks: - "src/src/third_party/mock_ocsp_responder/**" - "src/src/third_party/protobuf/**" - "src/src/third_party/schemastore.org/**" + - "src/tools/**" - "src/x509/**" exclude_files: - "src/*_test.pdb" diff --git a/poetry.lock b/poetry.lock index 71fc0ebfdd4..18b1e69107a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -972,15 +972,15 @@ devel = ["colorama", "json-spec", "jsonschema", "pylint", "pytest", "pytest-benc [[package]] name = "filelock" -version = "3.17.0" +version = "3.18.0" description = "A platform independent file lock." optional = false python-versions = ">=3.9" -groups = ["export"] +groups = ["export", "testing"] markers = "platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\"" files = [ - {file = "filelock-3.17.0-py3-none-any.whl", hash = "sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338"}, - {file = "filelock-3.17.0.tar.gz", hash = "sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e"}, + {file = "filelock-3.18.0-py3-none-any.whl", hash = "sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de"}, + {file = "filelock-3.18.0.tar.gz", hash = "sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2"}, ] [package.extras] @@ -5411,4 +5411,4 @@ libdeps = ["cxxfilt", "eventlet", "flask", "flask-cors", "gevent", "lxml", "prog [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "4353a9af69c54fed7b68a52c3896ae9793d81ce886fc5051762bd9e335ae17d8" +content-hash = "77da37f2c39364c35cf963b66852bb66e65e0d0c8fd20ee17aab0e3c8d1420ff" diff --git a/pyproject.toml b/pyproject.toml index 0621f2859af..d4804d0b320 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,6 +145,7 @@ ocspbuilder = "^0.10.2" ecdsa = "^0.19.0" asn1crypto = "^1.5.1" toml = ">=0.10.2,<0.11.0" +filelock = "^3.18.0" # Werkzeug is needed for ocsp tests in ocsp_mock.py # version 3+ fails with "ImportError: cannot import name 'url_quote' from 'werkzeug.urls'"