mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-97240 disable remote exec in CI (except for required + commit queue) (#30757) 

GitOrigin-RevId: 4282896240ca0c3fc8a50ab613421c098ce5a45a
This commit is contained in:
Daniel Moody 2025-01-02 21:23:56 -06:00 committed by MongoDB Bot
parent fb30b52fb9
commit d89e34eb19
17 changed files with 197 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.bazelrc
View File

@ -11,6 +11,9 @@ build --incompatible_enable_cc_toolchain_resolution
# about 1 minute on developer machines
build --experimental_remote_merkle_tree_cache
# allow multiple execution infos to be set
build --incompatible_modify_execution_info_additive
# This makes builds more hermetic by preventing environment variables from leaking into the execution of rules
build --incompatible_strict_action_env=true
@ -125,6 +128,9 @@ build --//bazel/config:build_enterprise=True
build --//bazel/config:release=False
build --//bazel/config:skip_archive=True
build --enable_platform_specific_config=true
build --jobs=200
build:macos --jobs=auto
build:windows --jobs=auto
# Profile for building fast with minimal debuggability - the build is fast ##################################################
build:fastbuild -c fastbuild
@ -250,7 +256,6 @@ build:windows --cxxopt=-UCOMPILER_MSVC
# Enable remote build execution:
build:linux --remote_executor=grpcs://sodalite.cluster.engflow.com
build:linux --jobs=200
# Restrict local actions to 20% to prevent linking from hitting OOM issues, but give the
# freedom to remote execution to schedule more compilation actions.

1
bazel/mongo_src_rules.bzl
View File

@ -1303,6 +1303,7 @@ MONGO_GLOBAL_SRC_DEPS = [
"//src/third_party/SafeInt:headers",
"//src/third_party/sasl:windows_sasl",
"//src/third_party/valgrind:headers",
"//src/third_party/abseil-cpp:absl_local_repo_deps",
]
MONGO_GLOBAL_DEFINES = (

23
bazel/platforms/BUILD.bazel
View File

@ -1,5 +1,5 @@
# Define supported build platforms.
load("//bazel/platforms:remote_execution_containers.bzl", "REMOTE_EXECUTION_CONTAINERS")
load("//bazel/platforms:platform_util.bzl", "setup_platform")
package(default_visibility = ["//visibility:public"])
@ -35,22 +35,7 @@ constraint_setting(name = "distro")
]
[
platform(
name = distro_or_os + "_" + arch,
constraint_values = [
"@platforms//os:linux",
"@platforms//cpu:arm64" if arch == "arm64" else "@platforms//cpu:x86_64",
":" + distro_or_os,
":use_mongo_toolchain",
],
exec_properties = {
"container-image": REMOTE_EXECUTION_CONTAINERS[distro_or_os]["container-url"],
"dockerNetwork": "standard",
# EngFlow's "default" pool is ARM64
"Pool": "x86_64" if arch == "amd64" else "default",
},
)
setup_platform(arch, distro_or_os, cache_silo)
for arch in [
"arm64",
"amd64",
@ -69,6 +54,10 @@ constraint_setting(name = "distro")
"rhel9",
"suse15",
]
for cache_silo in [
"",
"_cache_silo",
]
]
platform(

6
bazel/platforms/local_config_platform.bzl
View File

@ -44,6 +44,7 @@ def _setup_local_config_platform(ctx):
toolchain_exists = True
break
cache_silo = '"cache-silo-key": "' + toolchain_key + '",' if ctx.os.environ.get("evergreen_remote_exec") == "off" else ""
if ctx.os.environ.get("USE_NATIVE_TOOLCHAIN"):
exec_props = ""
result = {"USE_NATIVE_TOOLCHAIN": "1"}
@ -58,8 +59,9 @@ def _setup_local_config_platform(ctx):
"container-image": "%s",
"dockerNetwork": "standard",
"Pool": "%s",
%s
},
""" % (container_url, remote_execution_pool)
""" % (container_url, remote_execution_pool, cache_silo)
result = {"DISTRO": distro}
elif distro != None and toolchain_exists:
constraints_str += ',\n "@//bazel/platforms:use_mongo_toolchain"'
@ -95,7 +97,7 @@ def _setup_local_config_platform(ctx):
substitutions = substitutions,
)
return None
return result
setup_local_config_platform = repository_rule(
implementation = _setup_local_config_platform,

23
bazel/platforms/platform_util.bzl Normal file
View File

@ -0,0 +1,23 @@
load("//bazel/platforms:remote_execution_containers.bzl", "REMOTE_EXECUTION_CONTAINERS")
def setup_platform(arch, distro_or_os, cache_silo):
exec_properties = {
"container-image": REMOTE_EXECUTION_CONTAINERS[distro_or_os]["container-url"],
"dockerNetwork": "standard",
# EngFlow's "default" pool is ARM64
"Pool": "x86_64" if arch == "amd64" else "default",
}
if cache_silo:
exec_properties.update({"cache-silo-key": distro_or_os + "_" + arch})
native.platform(
name = distro_or_os + "_" + arch + cache_silo,
constraint_values = [
"@platforms//os:linux",
"@platforms//cpu:arm64" if arch == "arm64" else "@platforms//cpu:x86_64",
":" + distro_or_os,
":use_mongo_toolchain",
],
exec_properties = exec_properties,
)

4
etc/evergreen_yml_components/definitions.yml
View File

@ -1353,6 +1353,8 @@ functions:
type: test
params:
binary: bash
env:
evergreen_remote_exec: ${evergreen_remote_exec|off}
args:
- "src/evergreen/scons_compile.sh"
- command: expansions.update
@ -1442,6 +1444,8 @@ functions:
type: test
params:
binary: bash
env:
evergreen_remote_exec: ${evergreen_remote_exec|off}
args:
- "src/evergreen/bazel_compile.sh"

5
etc/evergreen_yml_components/variants/amazon/test_dev.yml
View File

@ -84,6 +84,7 @@ buildvariants:
expansions:
<<: *linux-arm64-static-enterprise-compile-expansions
compile_variant: *amazon-linux2-arm64-crypt-compile
evergreen_remote_exec: on
tasks:
- name: .crypt
- name: crypt_build_debug_and_test
@ -98,6 +99,7 @@ buildvariants:
build_mongot: true
download_mongot_release: true
compile_variant: *amazon-linux2-arm64-static-compile
evergreen_remote_exec: on
tasks:
- name: compile_ninja_quick_TG
- name: compile_test_parallel_unittest_stream_TG
@ -180,6 +182,7 @@ buildvariants:
scons_cache_mode: all
has_packages: false
skip_symbolization: true
evergreen_remote_exec: on
compile_flags: >-
--ssl
MONGO_DISTMOD=amazon2
@ -284,6 +287,7 @@ buildvariants:
-j$(grep -c ^processor /proc/cpuinfo)
--release
--variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
evergreen_remote_exec: on
run_on:
- amazon2-latest-xlarge
tasks:
@ -301,6 +305,7 @@ buildvariants:
-j$(grep -c ^processor /proc/cpuinfo)
--release
--variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
evergreen_remote_exec: on
run_on:
- amazon2-arm64-latest-xlarge
tasks:

3
etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml
View File

@ -121,8 +121,6 @@ buildvariants:
stepback: false
expansions:
compile_variant: *amazon-linux-2023-arm64-bazel-compile
compile_flags: >-
BAZEL_FLAGS="--config=remote_link"
tasks:
- name: compile_bazel_TG
- name: run_bazel_TG
@ -256,6 +254,7 @@ buildvariants:
--debug-symbols=minimal
separate_debug: off
scons_cache_scope: shared
evergreen_remote_exec: on
compile_variant: *al2023-arm64-sep-benchmark
tasks:
- name: .benchmarks_sep

2
etc/evergreen_yml_components/variants/sanitizer/test_dev.yml
View File

@ -225,6 +225,7 @@ buildvariants:
compile_variant: *linux-x86-dynamic-debug-compile-required
build_mongot: true
download_mongot_release: true
evergreen_remote_exec: on
test_flags: >-
--mongodSetParameters="{internalQueryEnableAggressiveSpillsInGroup: true}"
tasks:
@ -262,6 +263,7 @@ buildvariants:
--link-model=dynamic
build_mongot: true
download_mongot_release: true
evergreen_remote_exec: on
compile_variant: *linux-debug-aubsan-compile-required
test_flags: >-
--excludeWithAnyTags=incompatible_aubsan

2
etc/evergreen_yml_components/variants/ubuntu/test_dev_master_branch_only.yml
View File

@ -46,6 +46,7 @@ buildvariants:
stepback: false
expansions:
compile_variant: *workstation_bazel_cache_hydration_arm64
evergreen_remote_exec: on
tasks:
- name: hydrate_bazel_profile_TG
@ -58,5 +59,6 @@ buildvariants:
stepback: false
expansions:
compile_variant: *workstation_bazel_cache_hydration_x86_64
evergreen_remote_exec: on
tasks:
- name: hydrate_bazel_profile_TG

1
etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml
View File

@ -59,6 +59,7 @@ buildvariants:
<<: *linux_debug_community_compile_expansions
use_wt_develop: true
compile_variant: *linux-x86-dynamic-debug-wtdevelop-compile
evergreen_remote_exec: on
- name: linux-64-debug-wtdevelop
display_name: "~ Linux DEBUG WiredTiger develop"

4
evergreen/bazel_compile.sh
View File

@ -27,6 +27,10 @@ else
LOCAL_ARG="--config=local"
fi
if [[ "${evergreen_remote_exec}" != "on" ]]; then
LOCAL_ARG="$LOCAL_ARG --jobs=auto"
fi
ARCH=$(uname -m)
if [[ "$ARCH" == "arm64" || "$ARCH" == "aarch64" ]]; then
ARCH="arm64"

10
evergreen/generate_evergreen_bazelrc.sh
View File

@ -30,6 +30,16 @@ else
echo "build --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" >> .bazelrc.gitinfo
fi
source ./evergreen/bazel_RBE_supported.sh
if bazel_rbe_supported && [[ "${evergreen_remote_exec}" != "on" ]]; then
# Temporarily disable remote exec and only use remote cache
echo "common --remote_executor=" >> .bazelrc.evergreen
echo "common --modify_execution_info=.*=+no-remote-exec" >> .bazelrc.evergreen
echo "common --jobs=auto" >> .bazelrc.evergreen
echo "build:linux --local_resources=cpu=HOST_CPUS" >> .bazelrc.evergreen
fi
uri="https://spruce.mongodb.com/task/${task_id:?}?execution=${execution:?}"
echo "common --tls_client_certificate=./engflow.cert" >> .bazelrc.evergreen

22
site_scons/site_tools/integrate_bazel.py
View File

@ -1421,10 +1421,24 @@ def generate(env: SCons.Environment.Environment) -> None:
]
if not os.environ.get("USE_NATIVE_TOOLCHAIN"):
bazel_internal_flags += [
f"--platforms=//bazel/platforms:{distro_or_os}_{normalized_arch}",
f"--host_platform=//bazel/platforms:{distro_or_os}_{normalized_arch}",
]
if (
not is_local_execution(env)
and normalized_os == "linux"
and os.environ.get("evergreen_remote_exec") != "on"
):
cache_silo = "_cache_silo"
bazel_internal_flags += [
f"--platforms=//bazel/platforms:{distro_or_os}_{normalized_arch}{cache_silo}",
f"--host_platform=//bazel/platforms:{distro_or_os}_{normalized_arch}{cache_silo}",
"--spawn_strategy=local",
"--jobs=auto",
"--remote_executor=",
]
else:
bazel_internal_flags += [
f"--platforms=//bazel/platforms:{distro_or_os}_{normalized_arch}",
f"--host_platform=//bazel/platforms:{distro_or_os}_{normalized_arch}",
]
if tc := env.get("MONGO_TOOLCHAIN_VERSION"):
bazel_internal_flags += [f"--//bazel/config:mongo_toolchain_version={tc}"]

8
src/mongo/transport/grpc/BUILD.bazel
View File

@ -70,11 +70,6 @@ mongo_cc_library(
"util.h",
"wire_version_provider.h",
],
# At the moment, server.cpp depends directly on the gRPC-internal header
# "src/core/lib/security/credentials/ssl/ssl_credentials.h".
# Instead of adding it to the grpc overlay we explicitly add the include
# path for the header so that we don't leak the include globally.
copts = ["-Isrc/third_party/grpc/dist"],
target_compatible_with = select({
"//bazel/config:build_grpc_enabled": [],
"//conditions:default": ["@platforms//:incompatible"],
@ -204,6 +199,9 @@ mongo_cc_unit_test(
"//src/mongo/transport:session_workflow_test_util.h",
"//src/mongo/transport:transport_layer_manager_grpc_test.cpp",
],
data = [
"//jstests/libs:server_SAN.pem",
],
tags = ["mongo_unittest_first_group"],
target_compatible_with = GRPC_TARGET_COMPATIBLE_WITH,
deps = [

102
src/third_party/abseil-cpp/BUILD.bazel vendored
View File

@ -385,6 +385,108 @@ ABSEIL_HEADERS = [
"dist/absl/utility/utility.h",
]
mongo_cc_library(
name = "absl_local_repo_deps",
header_deps = select({
"//bazel/config:build_grpc_enabled": [
"@com_google_absl//absl/algorithm:algorithm",
"@com_google_absl//absl/algorithm:container",
"@com_google_absl//absl/base:base",
"@com_google_absl//absl/base:config",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/base:dynamic_annotations",
"@com_google_absl//absl/base:endian",
"@com_google_absl//absl/base:log_severity",
"@com_google_absl//absl/base:malloc_internal",
"@com_google_absl//absl/base:nullability",
"@com_google_absl//absl/base:prefetch",
"@com_google_absl//absl/cleanup:cleanup",
"@com_google_absl//absl/cleanup:cleanup_internal",
"@com_google_absl//absl/container:btree",
"@com_google_absl//absl/container:common",
"@com_google_absl//absl/container:compressed_tuple",
"@com_google_absl//absl/container:container_memory",
"@com_google_absl//absl/container:fixed_array",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_absl//absl/container:hash_policy_traits",
"@com_google_absl//absl/container:hashtable_debug",
"@com_google_absl//absl/container:hashtable_debug_hooks",
"@com_google_absl//absl/container:hashtablez_sampler",
"@com_google_absl//absl/container:inlined_vector",
"@com_google_absl//absl/container:inlined_vector_internal",
"@com_google_absl//absl/container:layout",
"@com_google_absl//absl/container:node_hash_map",
"@com_google_absl//absl/container:node_hash_set",
"@com_google_absl//absl/container:node_slot_policy",
"@com_google_absl//absl/container:raw_hash_map",
"@com_google_absl//absl/container:raw_hash_set",
"@com_google_absl//absl/crc:crc32c",
"@com_google_absl//absl/debugging:failure_signal_handler",
"@com_google_absl//absl/debugging:leak_check",
"@com_google_absl//absl/debugging:stacktrace",
"@com_google_absl//absl/debugging:symbolize",
"@com_google_absl//absl/flags:commandlineflag",
"@com_google_absl//absl/flags:config",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/flags:marshalling",
"@com_google_absl//absl/flags:parse",
"@com_google_absl//absl/flags:reflection",
"@com_google_absl//absl/flags:usage",
"@com_google_absl//absl/functional:any_invocable",
"@com_google_absl//absl/functional:bind_front",
"@com_google_absl//absl/functional:function_ref",
"@com_google_absl//absl/hash:city",
"@com_google_absl//absl/hash:hash",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/log:die_if_null",
"@com_google_absl//absl/log:flags",
"@com_google_absl//absl/log:globals",
"@com_google_absl//absl/log:initialize",
"@com_google_absl//absl/log:log",
"@com_google_absl//absl/log:log_entry",
"@com_google_absl//absl/log:log_sink",
"@com_google_absl//absl/log:log_sink_registry",
"@com_google_absl//absl/log:log_streamer",
"@com_google_absl//absl/log:structured",
"@com_google_absl//absl/memory:memory",
"@com_google_absl//absl/meta:type_traits",
"@com_google_absl//absl/numeric:bits",
"@com_google_absl//absl/numeric:int128",
"@com_google_absl//absl/numeric:representation",
"@com_google_absl//absl/random:bit_gen_ref",
"@com_google_absl//absl/random:distributions",
"@com_google_absl//absl/random:random",
"@com_google_absl//absl/random:seed_gen_exception",
"@com_google_absl//absl/random:seed_sequences",
"@com_google_absl//absl/status:status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings:cord",
"@com_google_absl//absl/strings:internal",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/strings:string_view",
"@com_google_absl//absl/strings:strings",
"@com_google_absl//absl/synchronization:synchronization",
"@com_google_absl//absl/time:time",
"@com_google_absl//absl/time/internal/cctz:civil_time",
"@com_google_absl//absl/time/internal/cctz:time_zone",
"@com_google_absl//absl/types:any",
"@com_google_absl//absl/types:bad_any_cast",
"@com_google_absl//absl/types:bad_optional_access",
"@com_google_absl//absl/types:bad_variant_access",
"@com_google_absl//absl/types:compare",
"@com_google_absl//absl/types:optional",
"@com_google_absl//absl/types:span",
"@com_google_absl//absl/types:variant",
"@com_google_absl//absl/utility:if_constexpr",
"@com_google_absl//absl/utility:utility",
],
"//conditions:default": [],
}),
)
ABSEIL_SKIP_GLOBAL_DEPS = [
# This is a globally injected dependency.
# Skip depending on all globally injected dependencies to avoid circular dependencies.

8
src/third_party/grpc/BUILD.bazel vendored
View File

@ -3154,11 +3154,13 @@ mongo_cc_library(
"dist/src/cpp/ext/proto_server_reflection.cc",
"dist/src/cpp/ext/proto_server_reflection_plugin.cc",
],
copts = GRPC_COPTS + [
"-Isrc/third_party/grpc/dist",
"-I$(BINDIR)/src/third_party/grpc/dist",
hdrs = [
"dist/src/core/lib/security/credentials/ssl/ssl_credentials.h",
"dist/src/cpp/ext/proto_server_reflection.h",
],
copts = GRPC_COPTS,
features = GRPC_FEATURES,
includes = ["dist"],
linkopts = ["-lssl"],
target_compatible_with = GRPC_TARGET_COMPATIBLE_WITH,
deps = [