From b7ea7576237bcc5f76115c73df4bdea73f7695f1 Mon Sep 17 00:00:00 2001 From: Andrew Bradshaw Date: Thu, 27 Mar 2025 10:17:17 -0700 Subject: [PATCH] SERVER-96986 Move CI from SCons to Bazel (#33596) GitOrigin-RevId: 1a2d45af69f48ab53f1d7db4c0848bba089f447a --- .bazelrc | 60 ++- BUILD.bazel | 114 +++- MODULE.bazel | 4 +- MODULE.bazel.lock | 6 +- WORKSPACE.bazel | 10 +- bazel/config/BUILD.bazel | 42 +- bazel/install_rules/install_rules.bzl | 154 +++++- bazel/install_rules/install_rules.py | 11 +- bazel/install_rules/pretty_printer_tests.bzl | 7 +- bazel/install_rules/windows_msvc.bzl | 8 + bazel/mongo_src_rules.bzl | 2 + bazel/platforms/local_config_platform.bzl | 2 + bazel/wrapper_hook/autogenerated_targets.py | 2 + buildscripts/bazel_scons_diff.py | 491 ----------------- buildscripts/generate_compile_expansions.py | 72 --- ...enerate_compile_expansions_shared_cache.py | 105 ---- .../suites/benchmarks_abt_lowering.yml | 6 +- .../resmokeconfig/suites/benchmarks_audit.yml | 4 +- .../suites/benchmarks_bsoncolumn.yml | 2 +- .../suites/benchmarks_expression.yml | 6 +- .../suites/benchmarks_expression_sbe.yml | 6 +- .../suites/benchmarks_first_half.yml | 30 +- .../suites/benchmarks_namespace_string.yml | 6 +- .../resmokeconfig/suites/benchmarks_query.yml | 36 +- .../suites/benchmarks_replication.yml | 2 +- .../suites/benchmarks_second_half.yml | 54 +- .../resmokeconfig/suites/benchmarks_sep.yml | 2 +- .../suites/benchmarks_sharding.yml | 2 +- .../suites/benchmarks_sorter.yml | 4 +- .../suites/benchmarks_storage.yml | 2 +- .../suites/benchmarks_streams.yml | 8 +- .../resmokeconfig/suites/benchmarks_util.yml | 4 +- .../suites/integration_tests_replset.yml | 9 +- .../integration_tests_replset_ssl_auth.yml | 7 +- .../suites/integration_tests_sharded.yml | 9 +- .../suites/integration_tests_standalone.yml | 8 +- .../integration_tests_standalone_audit.yml | 8 +- .../integration_tests_standalone_grpc.yml | 6 +- .../resmokeconfig/suites/libfuzzer.yml | 2 +- .../suites/pretty-printer-tests.yml | 2 +- .../resmokeconfig/suites/unittests.yml | 2 +- .../resmokeconfig/suites/unittests_auth.yml | 2 +- .../resmokeconfig/suites/unittests_client.yml | 2 +- .../suites/unittests_eighth_group.yml | 5 +- .../suites/unittests_fifth_group.yml | 2 +- .../suites/unittests_first_group.yml | 2 +- .../suites/unittests_fourth_group.yml | 2 +- .../resmokeconfig/suites/unittests_query.yml | 2 +- .../resmokeconfig/suites/unittests_repl.yml | 2 +- .../suites/unittests_second_group.yml | 5 +- .../suites/unittests_seventh_group.yml | 2 +- .../suites/unittests_sharding.yml | 2 +- .../suites/unittests_sixth_group.yml | 2 +- .../suites/unittests_third_group.yml | 2 +- buildscripts/resmokelib/config.py | 28 +- .../testing/hooks/metadata_consistency.py | 6 +- .../configuration.yml | 5 - etc/evergreen_yml_components/definitions.yml | 193 +------ .../tasks/compile_tasks.yml | 500 ++++++------------ .../tasks/compile_tasks_shared.yml | 140 ++--- .../tasks/misc_tasks.yml | 2 +- .../tasks/resmoke/non_server_teams/tasks.yml | 16 +- .../clusters_and_integrations/tasks.yml | 156 ++---- .../tasks.yml | 44 +- .../resmoke/server_divisions/query/tasks.yml | 22 +- .../variants/amazon/test_dev.yml | 140 ++--- .../amazon/test_dev_master_branch_only.yml | 61 +-- .../variants/amazon/test_release.yml | 157 ++---- .../variants/codecoverage/test_dev.yml | 35 +- .../variants/debian/test_release.yml | 26 +- ...t_release_master_and_lts_branches_only.yml | 90 +--- .../macos/test_dev_master_branch_only.yml | 8 +- .../variants/macos/test_release.yml | 38 +- .../variants/mongot/test_dev.yml | 48 +- .../variants/rhel/test_dev.yml | 14 +- .../test_dev_master_and_lts_branches_only.yml | 11 +- .../rhel/test_dev_master_branch_only.yml | 95 +--- .../variants/rhel/test_release.yml | 104 ++-- .../variants/sanitizer/test_dev.yml | 97 ++-- .../sanitizer/test_dev_master_branch_only.yml | 146 ++--- .../variants/suse/test_release.yml | 31 +- .../test_dev_master_and_lts_branches_only.yml | 55 +- .../ubuntu/test_dev_master_branch_only.yml | 17 - .../variants/ubuntu/test_release.yml | 162 ++---- .../variants/windows/test_dev.yml | 13 +- .../test_dev_master_and_lts_branches_only.yml | 13 +- .../windows/test_dev_master_branch_only.yml | 35 +- .../variants/windows/test_release.yml | 26 +- .../test_dev_master_branch_only.yml | 41 +- etc/expansions.default.yml | 8 +- evergreen/bazel_compile.sh | 42 +- evergreen/bazel_get_binary_path.sh | 9 - evergreen/bazel_scons_diff.sh | 41 -- evergreen/bazel_utility_functions.sh | 28 +- evergreen/crypt_run_tests.sh | 2 +- evergreen/failed_unittests_gather.sh | 2 +- evergreen/functions/bolt.sh | 6 +- .../functions/compile_expansions_generate.sh | 52 -- evergreen/functions/fix_tar.sh | 16 + .../shared_scons_directory_umount.sh | 19 - evergreen/generate_evergreen_bazelrc.sh | 7 +- evergreen/packager.py_run.sh | 2 +- evergreen/prelude_db_contrib_tool.sh | 2 + evergreen/scons_compile.sh | 196 ------- evergreen/scons_raw_compile.sh | 26 - evergreen/scons_splunk.sh | 6 - evergreen/stitch_support_create_lib_tar.sh | 9 - evergreen/stitch_support_run_tests.sh | 2 +- jstests/libs/hang_test_to_attach_gdb.js | 2 +- src/mongo/embedded/stitch_support/BUILD.bazel | 68 ++- src/third_party/wiredtiger/test/evergreen.yml | 2 +- 111 files changed, 1419 insertions(+), 3052 deletions(-) delete mode 100644 buildscripts/bazel_scons_diff.py delete mode 100755 buildscripts/generate_compile_expansions.py delete mode 100755 buildscripts/generate_compile_expansions_shared_cache.py delete mode 100644 evergreen/bazel_get_binary_path.sh delete mode 100644 evergreen/bazel_scons_diff.sh delete mode 100644 evergreen/functions/compile_expansions_generate.sh create mode 100755 evergreen/functions/fix_tar.sh delete mode 100755 evergreen/functions/shared_scons_directory_umount.sh delete mode 100755 evergreen/scons_compile.sh delete mode 100755 evergreen/scons_raw_compile.sh delete mode 100644 evergreen/scons_splunk.sh delete mode 100755 evergreen/stitch_support_create_lib_tar.sh diff --git a/.bazelrc b/.bazelrc index 763dcfbf284..9aced91abc2 100644 --- a/.bazelrc +++ b/.bazelrc @@ -16,6 +16,11 @@ common --enable_workspace=true # about 1 minute on developer machines common --experimental_remote_merkle_tree_cache +# Linter will have issues with duplicate configurations due to some of our +# install targets depending on tests that have testonly set. This flag can be removed +# if we stop installing our test targets. +common --experimental_retain_test_configuration_across_testonly + # allow multiple execution infos to be set common --incompatible_modify_execution_info_additive @@ -141,6 +146,7 @@ common --strip=never # To avoid the bazel warning about passing multiple configs common -c fastbuild common --fission=yes +common --//bazel/config:dwarf_version=5 common --//bazel/config:compress_debug_compile=True common --//bazel/config:opt=off common --//bazel/config:dbg=True @@ -150,7 +156,6 @@ common --//bazel/config:separate_debug=False common --//bazel/config:compiler_type=clang common --//bazel/config:linkstatic=True common --//bazel/config:build_enterprise=True -common --//bazel/config:release=False common --//bazel/config:skip_archive=True common --//bazel/config:allocator=auto common --//bazel/config:asan=False @@ -166,6 +171,7 @@ common --enable_platform_specific_config=true --config=fastbuild common:fastbuild -c fastbuild common:fastbuild --fission=yes +common:fastbuild --//bazel/config:dwarf_version=5 common:fastbuild --//bazel/config:compress_debug_compile=True common:fastbuild --//bazel/config:opt=off common:fastbuild --//bazel/config:dbg=True @@ -175,7 +181,6 @@ common:fastbuild --//bazel/config:separate_debug=False common:fastbuild --//bazel/config:compiler_type=clang common:fastbuild --//bazel/config:linkstatic=True common:fastbuild --//bazel/config:build_enterprise=True -common:fastbuild --//bazel/config:release=False common:fastbuild --//bazel/config:skip_archive=True common:fastbuild --//bazel/config:allocator=auto common:fastbuild --//bazel/config:asan=False @@ -188,6 +193,7 @@ common:fastbuild --enable_platform_specific_config=true --config=dbg common:dbg -c dbg common:dbg --fission=yes +common:dbg --//bazel/config:dwarf_version=5 common:dbg --//bazel/config:compress_debug_compile=True common:dbg --//bazel/config:opt=off common:dbg --//bazel/config:dbg=True @@ -197,7 +203,6 @@ common:dbg --//bazel/config:separate_debug=False common:dbg --//bazel/config:compiler_type=clang common:dbg --//bazel/config:linkstatic=True common:dbg --//bazel/config:build_enterprise=True -common:dbg --//bazel/config:release=False common:dbg --//bazel/config:skip_archive=True common:dbg --//bazel/config:allocator=auto common:dbg --//bazel/config:asan=False @@ -231,6 +236,7 @@ common:dbg_tsan --enable_platform_specific_config=true --config=opt common:opt -c opt common:opt --fission=yes +common:opt --//bazel/config:dwarf_version=5 common:opt --//bazel/config:compress_debug_compile=True common:opt --//bazel/config:opt=on common:opt --//bazel/config:dbg=False @@ -240,7 +246,6 @@ common:opt --//bazel/config:separate_debug=False common:opt --//bazel/config:compiler_type=gcc common:opt --//bazel/config:linkstatic=True common:opt --//bazel/config:build_enterprise=True -common:opt --//bazel/config:release=False common:opt --//bazel/config:skip_archive=True common:opt --//bazel/config:allocator=auto common:opt --//bazel/config:asan=False @@ -249,6 +254,40 @@ common:opt --//bazel/config:ubsan=False common:opt --//bazel/config:libunwind=auto common:opt --enable_platform_specific_config=true +--config=evg +common:evg --config=opt +common:evg --//bazel/config:opt=auto +common:evg --//bazel/config:separate_debug=True +common:evg --fission=no + +--config=evg_crypt +common:evg_crypt --config=evg +common:evg_crypt --//bazel/config:allocator=system +common:evg_crypt --//bazel/config:js_engine=none +common:evg_crypt --//bazel/config:ssl=False +common:evg_crypt --//bazel/config:http_client=False +common:evg_crypt --//bazel/config:shared_archive=True + +--config=evg_crypt_dbg +common:evg_crypt_dbg --config=evg_crypt +common:evg_crypt_dbg --//bazel/config:opt=off +common:evg_crypt_dbg --//bazel/config:dbg=True + +--config=evg_crypt_test +common:evg_crypt_test --config=evg_crypt +common:evg_crypt_test --//bazel/config:shared_archive=False + +--config=evg_stitch +common:evg_stitch --config=evg +common:evg_stitch --//bazel/config:build_enterprise=False +common:evg_stitch --//bazel/config:ssl=False +common:evg_stitch --//bazel/config:http_client=False +common:evg_stitch --//bazel/config:shared_archive=True + +--config=evg_stitch_test +common:evg_stitch_test --config=evg_stitch +common:evg_stitch_test --//bazel/config:shared_archive=False + # TODO: Build the code as we would release it # common:opt_release --config=opt # common:opt_release --//bazel/config:separate_debug=True @@ -301,6 +340,14 @@ common:windows --//bazel/config:linkstatic=True # The only Windows compiler we support is MSVC. common:windows --//bazel/config:compiler_type=msvc +# Always use clang on macos +common:macos --//bazel/config:compiler_type=clang + +common:macos --macos_minimum_os=11.0 +common:macos --use_libcxx=True +# TODO(SERVER-102959): Find out why tests fail when mac is not in compilation mode dbg +common:macos -c dbg + # Windows and MacOS do not use the custom toolchain config, so the compiler flags need to be set here. common:macos --cxxopt=-std=c++20 common:windows --cxxopt=/std:c++20 @@ -318,7 +365,9 @@ common:windows --cxxopt=-UCOMPILER_MSVC # the engflow cluster. External builders should use the --config=local option # Enable remote build execution: -common:linux --remote_executor=grpcs://sodalite.cluster.engflow.com +common --remote_executor=grpcs://sodalite.cluster.engflow.com +common:windows --remote_executor= +common:macos --remote_executor= # Enable remote cache (also necessary for remote build execution): common --remote_cache=grpcs://sodalite.cluster.engflow.com @@ -386,6 +435,7 @@ common:public-release --tls_client_key= common:public-release --remote_cache_compression=false common:public-release --grpc_keepalive_time=0s common:public-release --legacy_important_outputs +common:public-release --//bazel/config:release=True --config=fission common:fission --fission=yes diff --git a/BUILD.bazel b/BUILD.bazel index 26ab2696b6e..2d2c6bca210 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,14 +1,13 @@ load("@npm//:defs.bzl", "npm_link_all_packages") load("@aspect_rules_js//npm:defs.bzl", "npm_link_package") -load("//bazel/install_rules:install_rules.bzl", "mongo_install") +load("//bazel/install_rules:install_rules.bzl", "TEST_TAGS", "mongo_install") load("//bazel/toolchains:mongo_toolchain.bzl", "setup_mongo_toolchain_aliases") load("//bazel/config:render_template.bzl", "render_template") load("@npm//:eslint/package_json.bzl", eslint_bin = "bin") load("//bazel:mongo_js_rules.bzl", "mongo_js_library") load("@npm//:prettier/package_json.bzl", prettier = "bin") load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@aspect_rules_lint//format:defs.bzl", "format_multirun", "format_test") -load("//bazel/install_rules:install_rules.bzl", "TEST_TAGS") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") package( default_visibility = ["//visibility:public"], @@ -175,6 +174,59 @@ mongo_install( ], ) +mongo_install( + name = "mongocryptd", + srcs = [], + deps = [ + "//src/mongo/db/modules/enterprise:mongocryptd", + ], +) + +# This reflects the package that actually gets released during packaging. Be careful +# when changing this definition +mongo_install( + name = "dist", + srcs = [ + "//src/mongo/db:mongod", + "//src/mongo/installer/compass:compass_files", + "//src/mongo/s:mongos", + ] + select({ + "@platforms//os:windows": ["@local_windows_msvc//:vc_redist_x64"], + "//conditions:default": [], + }), + package_extract_name = select({ + "//bazel/config:build_enterprise_linux_enabled": "mongodb-linux-{TARGET_CPU}-enterprise-{MONGO_DISTMOD}-{MONGO_VERSION}", + "//bazel/config:build_enterprise_linux_disabled": "mongodb-linux-{TARGET_CPU}-{MONGO_DISTMOD}-{MONGO_VERSION}", + "//bazel/config:build_enterprise_windows_enabled": "mongodb-win32-{TARGET_CPU}-enterprise-{MONGO_DISTMOD}-{MONGO_VERSION}", + "//bazel/config:build_enterprise_windows_disabled": "mongodb-win32-{TARGET_CPU}-{MONGO_DISTMOD}-{MONGO_VERSION}", + "//bazel/config:build_enterprise_mac_enabled": "mongodb-macos-{TARGET_CPU}-enterprise-{MONGO_DISTMOD}-{MONGO_VERSION}", + "//bazel/config:build_enterprise_mac_disabled": "mongodb-macos-{TARGET_CPU}-{MONGO_DISTMOD}-{MONGO_VERSION}", + "//conditions:default": "mongodb-{TARGET_CPU}-{MONGO_DISTMOD}-{MONGO_VERSION}", + }), + publish_debug_in_stripped = select({ + "@platforms//os:windows": True, + "//conditions:default": False, + }), + deps = [ + "//src/mongo/db/modules/enterprise:dist", + ], +) + +filegroup( + name = "archive_license_files", + srcs = [ + "//distsrc:mpl", + "//distsrc:readme", + "//distsrc:third_party_notices", + ] + select({ + "//bazel/config:build_enterprise_enabled": ["//src/mongo/db/modules/enterprise/distsrc:enterprise_license"], + "//conditions:default": ["//distsrc:community_license"], + }) + select({ + "//bazel/config:build_enterprise_windows_enabled": ["//src/mongo/db/modules/enterprise/distsrc:windows_notices"], + "//conditions:default": [], + }), +) + copy_to_directory( name = "mongot_folder", srcs = select({ @@ -221,14 +273,19 @@ mongo_install( ) mongo_install( - name = "mongo-crypt", - srcs = [ - "//src/mongo/db:service_context_non_d", - "//src/mongo/db:service_context_test_fixture", - "//src/mongo/util/options_parser", - ], + name = "mongo_crypt", + srcs = [], + package_extract_name = "/", deps = [ - "//src/mongo/db/modules/enterprise:mongo-crypt", + "//src/mongo/db/modules/enterprise:mongo_crypt", + ], +) + +mongo_install( + name = "mongo_crypt_shlib_test", + srcs = [], + deps = [ + "//src/mongo/db/modules/enterprise:mongo_crypt_shlib_test", ], ) @@ -253,10 +310,28 @@ mongo_install( ], ) +# This is because the install system hardlinks files so permission changes end up +# being reflected on the source code version without this copy +copy_file( + name = "stitch_header", + src = "//src/mongo/embedded/stitch_support:stitch_support.h", + out = "copied_files/stitch_support.h", +) + mongo_install( - name = "stitch_support_dev", + name = "stitch_support", + srcs = ["//src/mongo/embedded/stitch_support"], + package_extract_name = "/", + root_files = { + "stitch_header": "include/stitch_support/v1/stitch_support", + }, +) + +mongo_install( + name = "stitch_support_test", + testonly = True, srcs = [ - "//src/mongo/embedded/stitch_support", + "//src/mongo/embedded/stitch_support:stitch_support_test", ], ) @@ -275,4 +350,19 @@ mongo_install( }), ) for target_name in TEST_TAGS.keys() + if target_name != "mongo_integration_test" ] + +mongo_install( + name = "mongo_integration_test", + testonly = True, + srcs = select({ + "//bazel/config:include_autogenerated_targets_enabled": [ + "//src/mongo/db:mongod", + "//src/mongo/db/modules/enterprise/autogenerated_targets:mongo_integration_test", + "//src/mongo/s:mongos", + "//src/mongo/shell:mongo", + ], + "//conditions:default": [], + }), +) diff --git a/MODULE.bazel b/MODULE.bazel index 1de8a7b1aed..98a19cac368 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -66,10 +66,10 @@ git_override( # ## We need skylib to be able to use config_setting_group in rule_poetry below ## https://github.com/bazelbuild/bazel-skylib/blob/main/docs/selects_doc.md#selectsconfig_setting_group -bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "bazel_skylib", version = "1.7.1") single_version_override( module_name = "bazel_skylib", - version = "1.5.0", + version = "1.7.1", ) bazel_dep(name = "rules_pkg", version = "0.9.1") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 56a9b05437f..2e5dca7a3ec 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -11,8 +11,8 @@ "https://bcr.bazel.build/modules/aspect_rules_js/2.1.3/source.json": "6b0fe67780c101430be087381b7a79d75eeebe1a1eae6a2cee937713603634ac", "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", "https://bcr.bazel.build/modules/bazel_features/1.10.0/source.json": "f7d074ea128c6e525157ae4becdc441c3f30581dcd5de367ec67277a1040729d", - "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", - "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/source.json": "77a1e1f3a18fcd0aa52822d7254e3d1fcc4a5509ca6eeaa922537762b452d314", + "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", + "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953", "https://bcr.bazel.build/modules/buildifier_prebuilt/6.4.0/MODULE.bazel": "37389c6b5a40c59410b4226d3bb54b08637f393d66e2fa57925c6fcf68e64bf4", "https://bcr.bazel.build/modules/buildifier_prebuilt/6.4.0/source.json": "83eb01b197ed0b392f797860c9da5ed1bf95f4d0ded994d694a3d44731275916", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", @@ -59,7 +59,7 @@ "moduleExtensions": { "//bazel:bzlmod.bzl%setup_mongo_python_toolchains": { "general": { - "bzlTransitiveDigest": "3lycEv/Lx6Dj7vGGPv6L+Q2hChjAqwIbmYW8dqPLlC8=", + "bzlTransitiveDigest": "K2FJu7KoZqpnHrorz3m629qCy0gc6MahFUDwzlEU7Jo=", "usagesDigest": "bUxjq9n+hj2YwYT/lcSP4lHyQ2GVy5JpFgSmddUqUZg=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 3f123566599..7a89343f9b0 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -40,7 +40,9 @@ filegroup( ) """, sha256 = "3e22e2b16f802277123590f64dfda44f1c9c8a2b7e758180cd956d8ab0965817", - url = "https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-2.1.28.zip", + urls = [ + "https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-2.1.28.zip", + ] * 5, ) http_archive( @@ -81,7 +83,9 @@ filegroup( ) """, sha256 = "6ac824e1642d6f7277d0ed7ea09411a508f6116ba6fae0aa5f2c7daa2ff43d31", - url = "https://github.com/wixtoolset/wix3/releases/download/wix3141rtm/wix314-binaries.zip", + urls = [ + "https://github.com/wixtoolset/wix3/releases/download/wix3141rtm/wix314-binaries.zip", + ] * 5, ) load("//bazel/install_rules:windows_msvc.bzl", "windows_msvc") @@ -177,7 +181,7 @@ new_local_repository( package(default_visibility = ["//visibility:public"]) filegroup( name = "mongot_binaries", - srcs = glob(["**"], exclude = ["BUILD", "WORKSPACE"]), + srcs = glob(["**"], exclude = ["BUILD.bazel", "WORKSPACE", "MODULE.bazel", "MODULE.bazel.lock"]), ) """, path = "mongot-localdev", diff --git a/bazel/config/BUILD.bazel b/bazel/config/BUILD.bazel index 1b68bece9a9..ad8fe38ff30 100644 --- a/bazel/config/BUILD.bazel +++ b/bazel/config/BUILD.bazel @@ -1141,7 +1141,7 @@ config_setting( use_ocsp_stapling( name = "use_ocsp_stapling", - build_setting_default = False, + build_setting_default = True, ) config_setting( @@ -1876,6 +1876,46 @@ selects.config_setting_group( ], ) +selects.config_setting_group( + name = "build_enterprise_windows_enabled", + match_all = [ + ":build_enterprise_enabled", + "@platforms//os:windows", + ], +) + +selects.config_setting_group( + name = "build_enterprise_mac_enabled", + match_all = [ + ":build_enterprise_enabled", + "@platforms//os:macos", + ], +) + +selects.config_setting_group( + name = "build_enterprise_linux_disabled", + match_all = [ + ":build_enterprise_disabled", + "@platforms//os:linux", + ], +) + +selects.config_setting_group( + name = "build_enterprise_windows_disabled", + match_all = [ + ":build_enterprise_disabled", + "@platforms//os:windows", + ], +) + +selects.config_setting_group( + name = "build_enterprise_mac_disabled", + match_all = [ + ":build_enterprise_disabled", + "@platforms//os:macos", + ], +) + # -------------------------------------- # streams non release options # -------------------------------------- diff --git a/bazel/install_rules/install_rules.bzl b/bazel/install_rules/install_rules.bzl index 02aef9f4b6f..1ad064221d2 100644 --- a/bazel/install_rules/install_rules.bzl +++ b/bazel/install_rules/install_rules.bzl @@ -2,7 +2,8 @@ Sets up install and archive rules. """ -load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@rules_pkg//:pkg.bzl", "pkg_tar", "pkg_zip") +load("@rules_pkg//:mappings.bzl", "pkg_attributes", "pkg_files") load("@rules_pkg//pkg:providers.bzl", "PackageFilesInfo") load("@bazel_skylib//lib:paths.bzl", "paths") load("//bazel:mongo_src_rules.bzl", "SANITIZER_DATA", "SANITIZER_ENV") @@ -10,6 +11,14 @@ load("//bazel:separate_debug.bzl", "TagInfo") load("//bazel/install_rules:pretty_printer_tests.bzl", "mongo_pretty_printer_test") load("//bazel/install_rules:providers.bzl", "TestBinaryInfo") +# Used to skip rules on certain OS architectures +def _empty_rule_impl(ctx): + pass + +empty_rule = rule( + implementation = _empty_rule_impl, +) + MongoInstallInfo = provider( doc = "A install rule provider to pass around deps files", fields = { @@ -40,6 +49,8 @@ TEST_TAGS = { "sharding_bm": 1, "sep_bm": 1, "storage_bm": 1, + "first_half_bm": 1, + "second_half_bm": 1, } def test_binary_aspect_impl(target, ctx): @@ -107,9 +118,9 @@ def is_binary_file(ctx, basename): """ linux_constraint, macos_constraint, windows_constraint = get_constraints(ctx) if ctx.target_platform_has_constraint(linux_constraint): - return not basename.startswith("lib") + return not (basename.startswith("lib") or basename.startswith("mongo_crypt_v") or basename.startswith("stitch_support.so")) elif ctx.target_platform_has_constraint(macos_constraint): - return not basename.startswith("lib") + return not (basename.startswith("lib") or basename.startswith("mongo_crypt_v") or basename.startswith("stitch_support.so")) elif ctx.target_platform_has_constraint(windows_constraint): return basename.endswith(".exe") or basename.endswith(".pdb") or basename.endswith(".dll") or basename.endswith(".ps1") else: @@ -173,14 +184,14 @@ def sort_file(ctx, file, install_dir, file_map, is_directory): if not is_debug_file(ctx, basename): if ctx.attr.debug != "debug": file_map["binaries"][file] = declare_output(ctx, bin_install, is_directory) - elif ctx.attr.debug != "stripped": + elif ctx.attr.debug != "stripped" or ctx.attr.publish_debug_in_stripped: file_map["binaries_debug"][file] = declare_output(ctx, bin_install, is_directory) elif not is_debug_file(ctx, basename): if ctx.attr.debug != "debug": file_map["dynamic_libs"][file] = declare_output(ctx, lib_install, is_directory) - elif ctx.attr.debug != "stripped": + elif ctx.attr.debug != "stripped" or ctx.attr.publish_debug_in_stripped: file_map["dynamic_libs_debug"][file] = declare_output(ctx, lib_install, is_directory) def mongo_install_rule_impl(ctx): @@ -200,6 +211,7 @@ def mongo_install_rule_impl(ctx): "binaries_debug": {}, "dynamic_libs_debug": {}, "dynamic_libs": {}, + "root_files": {}, } test_files = [] outputs = [] @@ -211,6 +223,10 @@ def mongo_install_rule_impl(ctx): for bin in input_bin.files.to_list(): sort_file(ctx, bin.path, install_dir, file_map, bin.is_directory) + for input_label, output_folder in ctx.attr.root_files.items(): + for file in input_label.files.to_list(): + file_map["root_files"][file.path] = declare_output(ctx, install_dir + "/" + output_folder + "/" + file.basename, file.is_directory) + # sort dependency install files for dep in ctx.attr.deps: test_files.extend(dep[TestBinaryInfo].test_binaries.to_list()) @@ -220,12 +236,17 @@ def mongo_install_rule_impl(ctx): src_map = json.decode(dep[MongoInstallInfo].src_map.to_list()[0]) files = [] for key in src_map: - files.extend(src_map[key]) + if key != "roots": + files.extend(src_map[key]) for file in files: - sort_file(ctx, file, install_dir, file_map, file_directory_map[file.split("/")[-1]]) + filename = file.split("/")[-1] + sort_file(ctx, file, install_dir, file_map, file_directory_map[filename]) + for file, folder in src_map["roots"].items(): + filename = file.split("/")[-1] + file_map["root_files"][file] = declare_output(ctx, install_dir + "/" + folder + "/" + filename, file_directory_map[filename]) # aggregate based on type of installs - if ctx.attr.debug == "stripped": + if ctx.attr.debug == "stripped" and not ctx.attr.publish_debug_in_stripped: bins = [bin for bin in file_map["binaries"]] libs = [lib for lib in file_map["dynamic_libs"]] elif ctx.attr.debug == "debug": @@ -234,9 +255,10 @@ def mongo_install_rule_impl(ctx): else: bins = [bin for bin in file_map["binaries"]] + [bin for bin in file_map["binaries_debug"]] libs = [lib for lib in file_map["dynamic_libs"]] + [lib for lib in file_map["dynamic_libs_debug"]] + root_files = [root_file for root_file in file_map["root_files"]] unittest_bin = None - if len(bins) == 1 and ctx.attr.debug != "debug" and file_map["binaries"][bins[0]].basename.endswith("_test"): + if len(bins) == 1 and ctx.attr.debug != "debug" and file_map["binaries"][bins[0]].basename.endswith("_test") and len(root_files) == 0: unittest_bin = file_map["binaries"][bins[0]] # Write installed_tests.txt which contains the list of all test files installed @@ -244,7 +266,13 @@ def mongo_install_rule_impl(ctx): installed_tests = [] for file in test_files: if not is_debug_file(ctx, file.basename) and ctx.attr.debug != "debug": - installed_tests.append(file_map["binaries"][file.path].path) + if is_binary_file(ctx, file.basename) or file.basename.endswith(".py"): + test_path = file_map["binaries"][file.path].path + + # point at the binaries in bazel-bin/install/ rather than bazel-out//bin// + split_test_path = test_path.split("/") + test_path = "bazel-bin/install/" + "/".join(split_test_path[4:]) + installed_tests.append(test_path) installed_test_list_file = None if len(installed_tests) > 0: @@ -254,16 +282,22 @@ def mongo_install_rule_impl(ctx): content = "\n".join(installed_tests), ) input_deps.append(installed_test_list_file) - real_test_list_output_location = ctx.actions.declare_file(install_dir + "/" + installed_test_list_file.basename) - outputs.append(real_test_list_output_location) # create a dep file for passing all the files we intend to install # to the python script name = ctx.label.package + "_" + install_dir name = name.replace("/", "_") deps_file = ctx.actions.declare_file("install_deps/" + name + "/" + install_dir) + + # The roots are in the format { file : folder } so we can add arbitrary files to the install directory + roots = {} if installed_test_list_file == None else {installed_test_list_file.path: ""} + for file in root_files: + path = file_map["root_files"][file].short_path + folder_index_start = path.find(install_dir) + len(install_dir) + 1 + folder_index_end = path.rfind("/") + roots[file] = path[folder_index_start:folder_index_end] json_out = struct( - roots = [] if installed_test_list_file == None else [installed_test_list_file.path], + roots = roots, bins = bins, libs = libs, ) @@ -284,6 +318,13 @@ def mongo_install_rule_impl(ctx): for file in libs: pkg_dict["lib/" + flat_map[file].basename] = flat_map[file] outputs.append(flat_map[file]) + for root_file in root_files: + pkg_dict[flat_map[root_file].basename] = flat_map[root_file] + outputs.append(flat_map[root_file]) + if len(installed_tests) > 0: + real_test_list_output_location = ctx.actions.declare_file(install_dir + "/" + installed_test_list_file.basename) + pkg_dict[real_test_list_output_location.basename] = real_test_list_output_location + outputs.append(real_test_list_output_location) # resolve full install dir for python script input full_install_dir = ctx.bin_dir.path @@ -296,7 +337,7 @@ def mongo_install_rule_impl(ctx): inputs = depset(direct = input_deps, transitive = [ ctx.attr._install_script.files, python.files, - ] + [f.files for f in ctx.attr.srcs] + [dep[MongoInstallInfo].deps_files for dep in ctx.attr.deps] + [dep[DefaultInfo].files for dep in ctx.attr.deps]) + ] + [f.files for f in ctx.attr.srcs] + [r.files for r in ctx.attr.root_files.keys()] + [dep[MongoInstallInfo].deps_files for dep in ctx.attr.deps] + [dep[DefaultInfo].files for dep in ctx.attr.deps]) if outputs: ctx.actions.run( @@ -348,6 +389,8 @@ mongo_install_rule = rule( "srcs": attr.label_list(aspects = [test_binary_aspect]), "deps": attr.label_list(providers = [PackageFilesInfo], aspects = [test_binary_aspect]), "debug": attr.string(), + "root_files": attr.label_keyed_string_dict(allow_files = True), + "publish_debug_in_stripped": attr.bool(), "_install_script": attr.label(allow_single_file = True, default = "//bazel/install_rules:install_rules.py"), "_linux_constraint": attr.label(default = "@platforms//os:linux"), "_macos_constraint": attr.label(default = "@platforms//os:macos"), @@ -361,9 +404,13 @@ def mongo_install( name, srcs, deps = [], + root_files = {}, target_compatible_with = [], testonly = False, pretty_printer_tests = {}, + archive_license_files = ["//:archive_license_files"], + package_extract_name = "dist-test", + publish_debug_in_stripped = False, **kwargs): """Perform install actions @@ -380,12 +427,6 @@ def mongo_install( "//conditions:default": None, }) - ext = select({ - "@platforms//os:linux": ".tar.gz", - "@platforms//os:windows": ".zip", - "@platforms//os:macos": ".tar.gz", - }) - # this macro create several install targets for each instance of an install: # "": normal install includes bins and debug info # stripped: only install bins, only available with separate_debug=True @@ -420,24 +461,42 @@ def mongo_install( # separate debug is required to make stripped or debug packages seperate_debug_incompat = [] - if install_type: - seperate_debug_incompat = ["@platforms//:incompatible"] + # TODO(SERVER-102851): This is commented out because CI AUBSAN builds currently + # try to build stripped/debug but don't separate them, that should be fixed + #if install_type: + # seperate_debug_incompat = ["@platforms//:incompatible"] if len(pretty_printer_tests) > 0 and install_type != "-debug": for test_script, test_binary in pretty_printer_tests.items(): pretty_printer_name = install_target + "-" + test_script.split(":")[-1].split(".")[0] mongo_pretty_printer_test( - name = pretty_printer_name, + name = "real_" + pretty_printer_name, test_script = test_script, test_binary = test_binary, testonly = True, ) + + # This is a hacky way to not produce the pretty printer test files on windows and + # mac - you can't use target_compatible_with because it will skip downstream rules, + # and in the downstream rules you can't use select or you have nested selects + empty_rule( + name = "fake_" + pretty_printer_name, + ) + + native.alias( + name = pretty_printer_name, + actual = select({ + "@platforms//os:linux": "real_" + pretty_printer_name, + "//conditions:default": "fake_" + pretty_printer_name, + }), + ) modified_srcs = modified_srcs + [pretty_printer_name] testonly = True mongo_install_rule( name = install_target, srcs = modified_srcs, + root_files = root_files, debug = debug, deps = select({ "//bazel/config:build_enterprise_enabled": dep_targets, @@ -447,16 +506,26 @@ def mongo_install( "//bazel/config:separate_debug_enabled": [], "//conditions:default": seperate_debug_incompat, }), + publish_debug_in_stripped = publish_debug_in_stripped, testonly = testonly, **kwargs ) + # This is so the README files dont end up looking like executables + pkg_files( + name = install_target + "_licenses", + srcs = archive_license_files, + attributes = pkg_attributes(mode = "644"), + ) + # package up the the install into an archive. pkg_tar( - name = "archive-" + name + install_type, - srcs = [install_target], + name = "archive-" + name + install_type + "_tar", + srcs = [install_target, install_target + "_licenses"], + mode = "755", compressor = compressor, - package_file_name = name + install_type + ext, + package_dir = package_extract_name, + package_file_name = name + install_type + ".tgz", exec_properties = { "no-cache": "1", "no-sandbox": "1", @@ -464,5 +533,38 @@ def mongo_install( "local": "1", }, testonly = testonly, + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), **kwargs ) + + pkg_zip( + name = "archive-" + name + install_type + "_zip", + srcs = [install_target, install_target + "_licenses"], + mode = "755", + package_dir = package_extract_name, + package_file_name = name + install_type + ".zip", + exec_properties = { + "no-cache": "1", + "no-sandbox": "1", + "no-remote": "1", + "local": "1", + }, + testonly = testonly, + target_compatible_with = select({ + "@platforms//os:windows": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + **kwargs + ) + + # Used to run zip on windows and tar on every other os + native.alias( + name = "archive-" + name + install_type, + actual = select({ + "@platforms//os:windows": "archive-" + name + install_type + "_zip", + "//conditions:default": "archive-" + name + install_type + "_tar", + }), + ) diff --git a/bazel/install_rules/install_rules.py b/bazel/install_rules/install_rules.py index c4482f0798d..599e6f8e51b 100644 --- a/bazel/install_rules/install_rules.py +++ b/bazel/install_rules/install_rules.py @@ -46,6 +46,7 @@ def install(src, install_type): if os.path.isdir(dst): shutil.rmtree(dst) else: + os.chmod(dst, 0o755) os.unlink(dst) except FileNotFoundError as exc: if link_dst == dst: @@ -68,7 +69,11 @@ def install(src, install_type): os.makedirs(dest_dir) os.link(os.path.join(root, name), os.path.join(dest_dir, name)) else: - os.link(src, dst) + try: + os.link(src, dst) + # If you try hardlinking across drives link will fail + except OSError: + shutil.copy(src, dst) except FileExistsError as exc: if link_dst == dst: # if multiple installs are requested at once and happen @@ -91,5 +96,5 @@ for depfile in args.depfile: install(binary, "bin") for lib in content["libs"]: install(lib, "lib") - for root in content["roots"]: - install(root, "") + for file, folder in content["roots"].items(): + install(file, folder) diff --git a/bazel/install_rules/pretty_printer_tests.bzl b/bazel/install_rules/pretty_printer_tests.bzl index 5a5eac47a8d..adecee5bb32 100644 --- a/bazel/install_rules/pretty_printer_tests.bzl +++ b/bazel/install_rules/pretty_printer_tests.bzl @@ -8,8 +8,7 @@ def mongo_pretty_printer_test_impl(ctx): # eg optimizer_gdb_test short_name = split_name[-1] - # eg /install-dist-test/bin/ - final_output_directory = "/" + "-".join(split_name[:-1]) + "/bin/" + final_output_directory = "bazel-bin/install/bin/" runnable_binary = None for file in ctx.attr.test_binary.files.to_list(): @@ -49,8 +48,8 @@ def mongo_pretty_printer_test_impl(ctx): "--gdb-path=" + "/opt/mongodbtoolchain/v4/bin/gdb", # This is due to us being dependent on the final location of installed binaries - ideally we don't do this and run the tests # in place and not from another directory - "--final-binary-path=" + ctx.bin_dir.path + final_output_directory + runnable_binary.basename, - "--final-pretty-printer-path=" + ctx.bin_dir.path + final_output_directory + script_output.basename, + "--final-binary-path=" + final_output_directory + runnable_binary.basename, + "--final-pretty-printer-path=" + final_output_directory + script_output.basename, "--pretty-printer-launcher-output=" + launcher_output.path, ], mnemonic = "MongoPrettyPrinterTestCreation", diff --git a/bazel/install_rules/windows_msvc.bzl b/bazel/install_rules/windows_msvc.bzl index c57788d9d00..c501360b59a 100644 --- a/bazel/install_rules/windows_msvc.bzl +++ b/bazel/install_rules/windows_msvc.bzl @@ -21,6 +21,14 @@ filegroup( "//conditions:default": [], }), ) + +filegroup( + name = "vc_redist_x64", + srcs = select({ + "@platforms//os:windows": glob(["**/vc_redist.x64.exe"]), + "//conditions:default": [], + }), +) """, ) else: diff --git a/bazel/mongo_src_rules.bzl b/bazel/mongo_src_rules.bzl index 823a09f2218..1643533e6a9 100644 --- a/bazel/mongo_src_rules.bzl +++ b/bazel/mongo_src_rules.bzl @@ -1690,6 +1690,7 @@ def mongo_cc_library( no_undefined_ref_DO_NOT_USE = True, linkshared = False, skip_windows_crt_flags = False, + shared_lib_name = "", **kwargs): """Wrapper around cc_library. @@ -1953,6 +1954,7 @@ def mongo_cc_library( user_link_flags = MONGO_GLOBAL_LINKFLAGS + package_specific_linkflags + undefined_ref_flag + non_transitive_dyn_linkopts + rpath_flags + visibility_support_shared_flags, target_compatible_with = shared_library_compatible_with + target_compatible_with + enterprise_compatible, dynamic_deps = dynamic_deps, + shared_lib_name = shared_lib_name, features = select({ "//bazel/config:windows_debug_symbols_enabled": ["generate_pdb_file"], "//conditions:default": [], diff --git a/bazel/platforms/local_config_platform.bzl b/bazel/platforms/local_config_platform.bzl index 7c23aeed593..73fbb329d03 100644 --- a/bazel/platforms/local_config_platform.bzl +++ b/bazel/platforms/local_config_platform.bzl @@ -51,6 +51,7 @@ def _setup_local_config_platform(ctx): result = {"USE_NATIVE_TOOLCHAIN": "1"} elif distro != None and distro in REMOTE_EXECUTION_CONTAINERS: constraints_str += ',\n "@//bazel/platforms:use_mongo_toolchain"' + constraints_str += ',\n "@//bazel/platforms:%s"' % (distro) container_url = REMOTE_EXECUTION_CONTAINERS[distro]["container-url"] web_url = REMOTE_EXECUTION_CONTAINERS[distro]["web-url"] dockerfile = REMOTE_EXECUTION_CONTAINERS[distro]["dockerfile"] @@ -66,6 +67,7 @@ def _setup_local_config_platform(ctx): result = {"DISTRO": distro} elif distro != None and toolchain_exists: constraints_str += ',\n "@//bazel/platforms:use_mongo_toolchain"' + constraints_str += ',\n "@//bazel/platforms:%s"' % (distro) result = {"DISTRO": distro} exec_props = "" else: diff --git a/bazel/wrapper_hook/autogenerated_targets.py b/bazel/wrapper_hook/autogenerated_targets.py index 047a6d6937b..e68ec3ee4da 100644 --- a/bazel/wrapper_hook/autogenerated_targets.py +++ b/bazel/wrapper_hook/autogenerated_targets.py @@ -20,6 +20,8 @@ bazel_tags_to_autogenerate = [ "sharding_bm", "sep_bm", "storage_bm", + "first_half_bm", + "second_half_bm", ] diff --git a/buildscripts/bazel_scons_diff.py b/buildscripts/bazel_scons_diff.py deleted file mode 100644 index 71e1b68716b..00000000000 --- a/buildscripts/bazel_scons_diff.py +++ /dev/null @@ -1,491 +0,0 @@ -# Generate linker commands in both Bazel and SCons for the intention of recording the differences -# during migration to Bazel. This will allow us to audit for differences if there are any issues -# found later on. - -import argparse -import os -import platform -import re -import subprocess -import sys - -from buildscripts.install_bazel import install_bazel - -bazel_env_settings: dict[str, str] = {} - - -def scons_to_bazel_target(scons_target: str) -> str: - # Example input: mongo/db/commands/libfsync_locked.a, output: //src/mongo/db/commands:fsync_locked - # Remove lib prefix in filename - scons_target = scons_target.replace("\\", "/") - scons_target_parts = scons_target.split("/") - if scons_target_parts[-1].startswith("lib"): - scons_target = "/".join(scons_target_parts[:-1] + [scons_target_parts[-1][3:]]) - bazel_target = f"//src/{scons_target}" - # Replace last / with : - last_slash_idx = bazel_target.rfind("/") - bazel_target = bazel_target[:last_slash_idx] + ":" + bazel_target[last_slash_idx + 1 :] - # Remove suffix - if "." in bazel_target: - bazel_target = bazel_target[: bazel_target.rfind(".")] - return bazel_target - - -def platformize_scons_target(scons_target: str) -> str: - # Patch the target name to match the platform's file naming conventions. - if platform.system() == "Windows": - scons_target = scons_target.replace(".so", ".dll") - scons_target = scons_target.replace(".a", ".lib") - if not scons_target.endswith(".lib") and not scons_target.endswith(".dll"): - scons_target += ".exe" - scons_target_parts = scons_target.split("/") - if scons_target_parts[-1].startswith("lib"): - scons_target = "/".join(scons_target_parts[:-1] + [scons_target_parts[-1][3:]]) - elif platform.system() == "Darwin": - scons_target = scons_target.replace(".so", ".dylib") - return scons_target - - -def normalize_link_mode_ext(scons_target: str, link_static: bool) -> str: - if link_static: - scons_target = scons_target.replace(".so", ".a") - scons_target = scons_target.replace(".dylib", ".a") - scons_target = scons_target.replace(".dll", ".lib") - else: - if platform.system() == "Darwin": - scons_target = scons_target.replace(".a", ".dylib") - else: - scons_target = scons_target.replace(".a", ".so") - scons_target = scons_target.replace(".dll", ".lib") - return scons_target - - -def log_subprocess_run(*args, **kwargs) -> subprocess.CompletedProcess: - arg_list_or_string = kwargs["args"] if "args" in kwargs else args[0] - print(" ".join(arg_list_or_string) if type(arg_list_or_string) == list else arg_list_or_string) - try: - proc = subprocess.run(*args, **kwargs) - print(proc.stdout) - print(proc.stderr) - return proc - except subprocess.CalledProcessError as e: - print(e.stdout) - print(e.stderr) - raise e - - -def get_bazel_args(compiler_type: str, extra_args: list[str]) -> list[str]: - # Do an actual run since dry-runs cannot create configure test entries which are needed for - # getting the command line flags from dry runs later in the execution of this script. - log_subprocess_run( - [ - sys.executable, - "buildscripts/scons.py", - *extra_args, - *( - [] - if compiler_type is None - else [f"--variables-files=etc/scons/mongodbtoolchain_stable_{compiler_type}.vars"] - ), - "VERBOSE=1", - "ICECC=", - "CCACHE=", - "--ninja=disabled", - "$BUILD_ROOT/scons/$VARIANT_DIR/sconf_temp", - ], - env={**os.environ.copy(), **bazel_env_settings}, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - text=True, - ) - with open("build/scons/bazel/bazel_command", "r") as bazel_command_file: - # strip off "bazel build" - return bazel_command_file.read().split(" ")[2:] - - -def bazel_compile_commands( - bazel_bin: str, bazel_targets: list[str], bazel_args: list[str] -) -> list[str]: - debug_targets = " ".join([f'"{target}_with_debug"' for target in bazel_targets]) - proc = log_subprocess_run( - [ - bazel_bin, - "aquery", - f'mnemonic("CppCompile", set({debug_targets}))', - "--include_artifacts=false", - *bazel_args, - ], - env={**os.environ.copy(), **bazel_env_settings}, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - text=True, - ) - - compile_lines_parts = [] - - in_command = False - for line in proc.stdout.split("\n"): - if "Command Line:" in line: - in_command = True - compile_lines_parts.append([]) - line = line.replace("Command Line:", "") - line = line.replace(" (exec ", "") - - if in_command: - if not line.endswith("\\"): - in_command = False - - # Remove the trailing \ or ) - if line.endswith("\\") or line.endswith(")"): - line = line[:-1] - compile_lines_parts[-1] += [line.strip()] - - bazel_output_files = set() - compile_lines = [] - for i, compile_line_parts in enumerate(compile_lines_parts): - # Move the -o {file} part to the beginning - compiler_path = compile_line_parts[:1] - if platform.system() == "Windows": - # Windows appends /c - output_args = [compile_line_parts[-3]] - other_args = compile_line_parts[1:-3] + compile_line_parts[-2:] - else: - output_args = compile_line_parts[-2:] - other_args = compile_line_parts[1:-2] - if output_args[0].startswith("/Fo"): - output_args[0] = output_args[0].replace("/Fo", "") - compile_line_parts = compiler_path + output_args + other_args - - bazel_output_files.add(compile_line_parts[-1]) - - compile_lines += [" ".join(compile_line_parts)] - - # Replace Bazel parts to match SCons output - compile_lines[i] = re.sub(r"bazel-out/\S*/bin/src/", "", compile_lines[i]) - compile_lines[i] = re.sub(r"_objs/(\S*)_with_debug", r"\1", compile_lines[i]) - - with open("./build/bazel-compile-commands", "w") as output_file: - # Sort by output path - compile_lines = sorted(compile_lines, key=lambda compile_line: compile_line.split(" ")[1]) - for compile_line in compile_lines: - print(compile_line, file=output_file) - - return bazel_output_files - - -def bazel_linker_commands( - bazel_bin: str, - bazel_targets: list[str], - target_type_map: dict[str, str], - bazel_args: list[str], - link_static: bool, -) -> list[str]: - debug_targets = " ".join( - [ - f'"{target}_shared_with_debug"' - if target_type_map[target] == "library" and not link_static - else f'"{target}_with_debug"' - for target in bazel_targets - ] - ) - proc = log_subprocess_run( - [ - bazel_bin, - "aquery", - f'mnemonic("CppLink", set({debug_targets}))', - "--include_artifacts=false", - *bazel_args, - ], - env={**os.environ.copy(), **bazel_env_settings}, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - text=True, - ) - - link_lines_parts = [] - - in_command = False - for line in proc.stdout.split("\n"): - if "Command Line: (exec " in line: - in_command = True - link_lines_parts.append([]) - line = line.replace("Command Line: (exec ", "") - - if in_command: - if not line.endswith("\\"): - in_command = False - - # Remove the trailing \ or ) - line = line[:-1] - link_lines_parts[-1] += [line.strip()] - - linker_output_paths = set() - link_lines = [] - for i, link_line_parts in enumerate(link_lines_parts): - # Remove the compiler path to match SCons output - link_line_parts = link_line_parts[1:] - - # Move the -shared flag to the end to match SCons output - if link_line_parts[0] == "-shared": - link_line_parts = link_line_parts[1:] + link_line_parts[:1] - - link_lines += [" ".join(link_line_parts)] - - # Cleanup spammy linker flags - link_lines[i] = re.sub(r"-Xlinker -rpath -Xlinker \S* ", "", link_lines[i]) - - # Replace bazel paths to match SCons paths - link_lines[i] = re.sub(r"bazel-out/\S*/bin/src/", "", link_lines[i]) - link_lines[i] = re.sub(r"_shared_with_debug", "", link_lines[i]) - link_lines[i] = re.sub(r"_with_debug", "", link_lines[i]) - - linker_output_paths.update([link_line.split(" ")[1] for link_line in link_lines]) - - with open("./build/bazel-link-commands", "w") as output_file: - # Sort by output path - link_lines = sorted(link_lines, key=lambda link_line: link_line.split(" ")[1]) - for link_line in link_lines: - print(link_line, file=output_file) - return linker_output_paths - - -def scons_commands( - scons_targets: list[str], - compiler_type: str, - compile_output_paths: set[str], - linker_output_paths: set[str], - extra_args: list[str], -): - log_subprocess_run( - [ - sys.executable, - "buildscripts/scons.py", - *extra_args, - *( - [] - if compiler_type is None - else [f"--variables-files=etc/scons/mongodbtoolchain_stable_{compiler_type}.vars"] - ), - "VERBOSE=1", - "ICECC=", - "CCACHE=", - "--ninja=disabled", - "$BUILD_ROOT/scons/$VARIANT_DIR/sconf_temp", - ], - env={**os.environ.copy(), **bazel_env_settings}, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - text=True, - ) - proc = log_subprocess_run( - [ - sys.executable, - "buildscripts/scons.py", - *extra_args, - *( - [] - if compiler_type is None - else [f"--variables-files=etc/scons/mongodbtoolchain_stable_{compiler_type}.vars"] - ), - "VERBOSE=1", - "ICECC=", - "CCACHE=", - "--ninja=disabled", - "--dry-run", - *scons_targets, - ], - env={**os.environ.copy(), **bazel_env_settings}, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - text=True, - ) - - with open("./build/scons-compile-commands", "w") as output_file: - # Use -fno-omit-frame-pointer and /D_CRT_SECURE_NO_WARNINGS to identify compile commands since they're present on all compilations and will - # not be removed in the lifetime of the hybrid build system. - compile_commands = [ - output_line - for output_line in proc.stdout.split("\n") - if ("-fno-omit-frame-pointer" in output_line and "-o" in output_line) - or "/D_CRT_SECURE_NO_WARNINGS" in output_line - ] - - # Replace prefix to match Bazel - compile_commands = [ - re.sub(r"build[\\/](opt|debug|san|optdebug)[\\/]", "", compile_command) - for compile_command in compile_commands - ] - - # Sort by output path - compile_commands = sorted( - compile_commands, key=lambda compile_command: compile_command.split(" ")[2] - ) - - compile_commands = [ - " ".join(compile_command.split(" ")) for compile_command in compile_commands - ] - - for compile_command in compile_commands: - # Skip over any lines with inputs not used in the Bazel compilation - src_path = ( - compile_command.split(" ")[3].replace("\\", "/") - if platform.system() == "Windows" - else compile_command.split(" ")[-1] - ) - if src_path not in compile_output_paths: - continue - print(compile_command, file=output_file) - - with open("./build/scons-link-commands", "w") as output_file: - link_commands = [ - output_line - for output_line in proc.stdout.split("\n") - if ("-Wl," in output_line and "-o" in output_line) - or "/LARGEADDRESSAWARE" in output_line - ] - - # Remove binary name since it's not present in Bazel - link_commands = [" ".join(link_command.split(" ")[1:]) for link_command in link_commands] - - # Replace prefix to match Bazel - link_commands = [ - re.sub(r"build[\\/](opt|debug|san|optdebug)[\\/]", "", link_command) - for link_command in link_commands - ] - - # Sort by output path - link_commands = sorted(link_commands, key=lambda link_command: link_command.split(" ")[1]) - - link_commands = [" ".join(link_command.split(" ")) for link_command in link_commands] - - for link_command in link_commands: - # Skip over any lines with outputs not referenced in the Bazel compilation - if link_command.split(" ")[1] not in linker_output_paths: - continue - print(link_command, file=output_file) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--compiler_type", - type=str, - help="compiler type override, use for running locally", - default=None, - ) - parser.add_argument("--extra_args", type=str, help="list of args to pass to scons", default="") - parser.add_argument( - "scons_targets", - nargs="+", - help="List of SCons targets to compare with their Bazel equivalents. Remove the build/*config/ prefix. " - + "Example: for build/fast/mongo/platform/visibility_test1 pass in mongo/platform/visibility_test1", - ) - - args = parser.parse_args() - - # Needed for git stash on Windows - git_env = { - **os.environ.copy(), - **{ - "GIT_COMMITTER_NAME": "Evergreen", - "GIT_COMMITTER_EMAIL": "evergreen@mongodb.com", - "GIT_AUTHOR_NAME": "Evergreen", - "GIT_AUTHOR_EMAIL": "evergreen@mongodb.com", - }, - } - - # Switch to repository root directory. - os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - - # Set JAVA_HOME on ppc & s390x architectures - global bazel_env_settings - if platform.machine().lower() in {"ppc64le", "s390x"}: - bazel_env_settings["JAVA_HOME"] = "/usr/lib/jvm/java-21-openjdk" - - target_type_map = {} - for scons_target in args.scons_targets: - basename = os.path.basename(scons_target) - if basename.endswith((".a", ".lib", ".dylib", ".dll", ".so")): - target_type = "library" - elif (basename.endswith(".exe") and platform.system() == "Windows") or ( - "." not in (basename and platform.system() != "Windows") - ): - target_type = "binary" - else: - print("malformed target:", scons_target) - exit(1) - target_type_map[scons_to_bazel_target(scons_target)] = target_type - - extra_args = args.extra_args.strip().split(" ") if args.extra_args != "" else [] - - # Replace the single quotes in the build command that would usually be removed by Bash - extra_args = [extra_arg.replace("'", "") for extra_arg in extra_args] - bazel_args = get_bazel_args(args.compiler_type, extra_args) - link_static = "--//bazel/config:linkstatic=True" in bazel_args - - scons_targets = [ - "$BUILD_DIR/" - + normalize_link_mode_ext(platformize_scons_target(scons_target), link_static).replace( - "\\", "/" - ) - for scons_target in args.scons_targets - ] - bazel_targets = [scons_to_bazel_target(scons_target) for scons_target in args.scons_targets] - - print("Bazel targets:", bazel_targets) - print("SCons targets:", scons_targets) - - bazel_bin_dir = ( - os.getenv("TMPDIR") if os.getenv("TMPDIR") else os.path.expanduser("~/.local/bin") - ) - if not os.path.exists(bazel_bin_dir): - os.makedirs(bazel_bin_dir) - - bazel_bin = install_bazel(bazel_bin_dir) - print("Bazel bin:", bazel_bin_dir) - - compile_output_paths = bazel_compile_commands(bazel_bin, bazel_targets, bazel_args) - linker_output_paths = bazel_linker_commands( - bazel_bin, bazel_targets, target_type_map, bazel_args, link_static - ) - - # With thin targets, the build system only allows one definition of each target between both SCons and Bazel. - # Since we want to get a diff after removing the target from SCons and adding it to Bazel, we can rely on git - # to revert the change to add the BUILD.bazel definitions when we want to execute the SCons version of the build. - current_branch = subprocess.check_output( - ["git", "rev-parse", "--abbrev-ref", "HEAD"], env=git_env, text=True - ).strip() - subprocess.run(["git", "commit", "-m", "tmp"], env=git_env) - subprocess.run(["git", "checkout", "HEAD~1"], env=git_env, check=True) - scons_commands( - scons_targets, args.compiler_type, compile_output_paths, linker_output_paths, extra_args - ) - - with open("./build/merged_diff.md", "w") as output_file: - output_file.write(" ".join(sys.argv) + "\n\n") - for file_name in [ - "scons-compile-commands", - "bazel-compile-commands", - "scons-link-commands", - "bazel-link-commands", - ]: - with open(f"./build/{file_name}", "r") as input_file: - output_file.write(f"{file_name}:\n\n") - print(f"{file_name}:\n\n") - output_file.write("```\n") - for line in input_file: - output_file.write(line) - print(line) - print("\n\n\n") - output_file.write("\n```\n\n\n") - subprocess.run(["git", "checkout", current_branch], env=git_env, check=True) - subprocess.run(["git", "reset", "HEAD~1"], env=git_env, check=True) - - -if __name__ == "__main__": - main() diff --git a/buildscripts/generate_compile_expansions.py b/buildscripts/generate_compile_expansions.py deleted file mode 100755 index 2be145f1911..00000000000 --- a/buildscripts/generate_compile_expansions.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -""" -Generate the compile expansions file used by Evergreen as part of the push/release process. - -Invoke by specifying an output file. -$ python generate_compile_expansions.py --out compile_expansions.yml -""" - -import argparse -import os -import shlex -import sys - -import yaml - -VERSION_JSON = "version.json" - - -def generate_expansions(): - """Entry point for the script. - - This calls functions to generate version and scons cache expansions and - writes them to a file. - """ - args = parse_args() - expansions = {} - expansions.update(generate_scons_cache_expansions()) - - with open(args.out, "w") as out: - print("saving compile expansions to {0}: ({1})".format(args.out, expansions)) - yaml.safe_dump(expansions, out, default_flow_style=False) - - -def parse_args(): - """Parse program arguments.""" - parser = argparse.ArgumentParser() - parser.add_argument("--out", required=True) - return parser.parse_args() - - -def generate_scons_cache_expansions(): - """Generate scons cache expansions from some files and environment variables.""" - expansions = {} - if sys.platform.startswith("win"): - system_id_path = r"c:\mongodb-build-system-id" - default_cache_path_base = r"z:\data\scons-cache" - else: - system_id_path = "/etc/mongodb-build-system-id" - default_cache_path_base = "/data/scons-cache" - - if os.path.isfile(system_id_path): - with open(system_id_path, "r") as fh: - default_cache_path = os.path.join(default_cache_path_base, fh.readline().strip()) - - expansions["scons_cache_path"] = default_cache_path - - scons_cache_mode = os.getenv("SCONS_CACHE_MODE") - - if scons_cache_mode in (None, ""): - scons_cache_mode = "nolinked" - - if os.getenv("USE_SCONS_CACHE") not in (None, False, "false", ""): - expansions["scons_cache_args"] = ( - "--cache={0} --cache-signature-mode=validate --cache-dir={1} --cache-show".format( - scons_cache_mode, shlex.quote(default_cache_path) - ) - ) - return expansions - - -if __name__ == "__main__": - generate_expansions() diff --git a/buildscripts/generate_compile_expansions_shared_cache.py b/buildscripts/generate_compile_expansions_shared_cache.py deleted file mode 100755 index f22e0d9da77..00000000000 --- a/buildscripts/generate_compile_expansions_shared_cache.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 -""" -Generate the compile expansions file used by Evergreen as part of the push/release process. - -Invoke by specifying an output file. -$ python generate_compile_expansions.py --out compile_expansions.yml -""" - -import argparse -import os -import shlex -import sys - -import yaml - -VERSION_JSON = "version.json" - - -def generate_expansions(): - """Entry point for the script. - - This calls functions to generate version and scons cache expansions and - writes them to a file. - """ - args = parse_args() - expansions = {} - expansions.update(generate_scons_cache_expansions()) - - with open(args.out, "w") as out: - print("saving compile expansions to {0}: ({1})".format(args.out, expansions)) - yaml.safe_dump(expansions, out, default_flow_style=False) - - -def parse_args(): - """Parse program arguments.""" - parser = argparse.ArgumentParser() - parser.add_argument("--out", required=True) - return parser.parse_args() - - -def generate_scons_cache_expansions(): - """Generate scons cache expansions from some files and environment variables.""" - expansions = {} - - # Get the scons cache mode - scons_cache_mode = os.getenv("SCONS_CACHE_MODE", "nolinked") - - # Get the host uuid - if sys.platform.startswith("win"): - system_id_path = r"c:\mongodb-build-system-id" - else: - system_id_path = "/etc/mongodb-build-system-id" - - if os.path.isfile(system_id_path): - with open(system_id_path, "r") as fh: - system_uuid = fh.readline().strip() - - # Set the scons shared cache setting - - # Global shared cache using EFS - if os.getenv("SCONS_CACHE_SCOPE") == "shared": - if sys.platform.startswith("win"): - shared_mount_root = "X:\\" - else: - shared_mount_root = "/efs/scons" - - scons_cache_dir = os.getenv("SCONS_CACHE_DIR") - if scons_cache_dir: - default_cache_path = os.path.join( - shared_mount_root, system_uuid, "per_variant_caches", scons_cache_dir, "scons-cache" - ) - else: - default_cache_path = os.path.join(shared_mount_root, system_uuid, "scons-cache") - - expansions["scons_cache_path"] = default_cache_path - expansions["scons_cache_args"] = ( - "--cache=nolinked --cache-signature-mode=validate --cache-dir={0} --cache-show".format( - shlex.quote(default_cache_path) - ) - ) - - # Local shared cache - host-based - elif os.getenv("SCONS_CACHE_SCOPE") == "local": - if sys.platform.startswith("win"): - default_cache_path_base = r"z:\data\scons-cache" - else: - default_cache_path_base = "/data/scons-cache" - - default_cache_path = os.path.join(default_cache_path_base, system_uuid) - expansions["scons_cache_path"] = default_cache_path - expansions["scons_cache_args"] = ( - "--cache={0} --cache-signature-mode=validate --cache-dir={1} --cache-show".format( - scons_cache_mode, shlex.quote(default_cache_path) - ) - ) - # No cache - else: - # Anything else is 'none' - print("No cache used") - - return expansions - - -if __name__ == "__main__": - generate_expansions() diff --git a/buildscripts/resmokeconfig/suites/benchmarks_abt_lowering.yml b/buildscripts/resmokeconfig/suites/benchmarks_abt_lowering.yml index 239369454b2..8988434cfcf 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_abt_lowering.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_abt_lowering.yml @@ -2,11 +2,11 @@ test_kind: benchmark_test selector: - root: build/benchmarks.txt + root: bazel-bin/install/install-mongo_benchmark-stripped_test_list.txt include_files: # The trailing asterisk is for handling the .exe extension on Windows. - - build/**/system_resource_canary_bm* - - build/install/bin/abt_lower_bm* + - bazel-bin/**/system_resource_canary_bm* + - bazel-bin/**/abt_lower_bm* executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_audit.yml b/buildscripts/resmokeconfig/suites/benchmarks_audit.yml index 86609907586..f9a99a66178 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_audit.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_audit.yml @@ -1,10 +1,10 @@ test_kind: benchmark_test selector: - root: build/benchmarks.txt + root: bazel-bin/install/install-mongo_benchmark-stripped_test_list.txt include_files: # The trailing asterisk is for handling the .exe extension on Windows. - - build/install/bin/audit_bm* + - bazel-bin/**/audit_bm* executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_bsoncolumn.yml b/buildscripts/resmokeconfig/suites/benchmarks_bsoncolumn.yml index 498e8f246b2..79dbb8f190b 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_bsoncolumn.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_bsoncolumn.yml @@ -2,7 +2,7 @@ test_kind: benchmark_test selector: - root: build/bsoncolumn_bm.txt + root: bazel-bin/install/install-bsoncolumn_bm_test_list.txt executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_expression.yml b/buildscripts/resmokeconfig/suites/benchmarks_expression.yml index 85d61376173..8cb2e0d3f2e 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_expression.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_expression.yml @@ -2,11 +2,11 @@ test_kind: benchmark_test selector: - root: build/benchmarks.txt + root: bazel-bin/install/install-mongo_benchmark-stripped_test_list.txt include_files: # The trailing asterisk is for handling the .exe extension on Windows. - - build/**/system_resource_canary_bm* - - build/install/bin/expression_bm* + - bazel-bin/**/system_resource_canary_bm* + - bazel-bin/**/expression_bm* executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_expression_sbe.yml b/buildscripts/resmokeconfig/suites/benchmarks_expression_sbe.yml index 7f8d178dae0..50f82f9114d 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_expression_sbe.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_expression_sbe.yml @@ -2,11 +2,11 @@ test_kind: benchmark_test selector: - root: build/benchmarks.txt + root: bazel-bin/install/install-mongo_benchmark-stripped_test_list.txt include_files: # The trailing asterisk is for handling the .exe extension on Windows. - - build/**/system_resource_canary_bm* - - build/install/bin/sbe_expression_bm* + - bazel-bin/**/system_resource_canary_bm* + - bazel-bin/**/sbe_expression_bm* executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_first_half.yml b/buildscripts/resmokeconfig/suites/benchmarks_first_half.yml index 40386f2e7ad..c5168dc3e41 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_first_half.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_first_half.yml @@ -1,32 +1,32 @@ test_kind: benchmark_test selector: - root: build/first_half_bm.txt + root: bazel-bin/install/install-first_half_bm_test_list.txt exclude_files: # The trailing asterisk is for handling the .exe extension on Windows. # Hash table benchmark is really slow, don't run on evergreen - - build/install/bin/hash_table_bm* + - bazel-bin/**/hash_table_bm* # These benchmarks are being run as part of the benchmarks_query.yml - - build/install/bin/query_bm* - - build/install/bin/plan_cache_classic_bm* - - build/install/bin/point_query_bm* - - build/install/bin/distinct_scan_bm* + - bazel-bin/**/query_bm* + - bazel-bin/**/plan_cache_classic_bm* + - bazel-bin/**/point_query_bm* + - bazel-bin/**/distinct_scan_bm* # These benchmarks are being run as part of the benchmarks_expression*.yml - - build/install/bin/expression_bm* - - build/install/bin/sbe_expression_bm* + - bazel-bin/**/expression_bm* + - bazel-bin/**/sbe_expression_bm* # These benchmarks are being run as part of the benchmarks_streams.yml test suite. - - build/install/bin/streams_operator_dag_bm* - - build/install/bin/streams_window_operator_bm* + - bazel-bin/**/streams_operator_dag_bm* + - bazel-bin/**/streams_window_operator_bm* # These benchmarks are only run when modifying or upgrading the immutable library. - - build/install/bin/immutable_absl_comparison_bm* - - build/install/bin/immutable_std_comparison_bm* + - bazel-bin/**/immutable_absl_comparison_bm* + - bazel-bin/**/immutable_std_comparison_bm* # These benchmarks are being run as part of the benchmarks_util.yml test suite. - - build/install/bin/tracking_allocator_bm* + - bazel-bin/**/tracking_allocator_bm* # These benchmarks are being run as part of the benchmarks_sorter.yml test suite. - - build/install/bin/sorter_checksum_calculator_bm* + - bazel-bin/**/sorter_checksum_calculator_bm* # These benchmarks are being run as part of the benchmarks_audit.yml test suite. - - build/install/bin/audit_bm* + - bazel-bin/**/audit_bm* executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_namespace_string.yml b/buildscripts/resmokeconfig/suites/benchmarks_namespace_string.yml index ba77848b8dc..74ff2ea74cb 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_namespace_string.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_namespace_string.yml @@ -2,11 +2,11 @@ test_kind: benchmark_test selector: - root: build/benchmarks.txt + root: bazel-bin/install/install-mongo_benchmark-stripped_test_list.txt include_files: # The trailing asterisk is for handling the .exe extension on Windows. - - build/**/system_resource_canary_bm* - - build/install/bin/namespace_string_bm* + - bazel-bin/**/system_resource_canary_bm* + - bazel-bin/**/namespace_string_bm* executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_query.yml b/buildscripts/resmokeconfig/suites/benchmarks_query.yml index 6c96bdee990..7f5383e8886 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_query.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_query.yml @@ -2,26 +2,26 @@ test_kind: benchmark_test selector: - root: build/benchmarks.txt + root: bazel-bin/install/install-query_bm_test_list.txt include_files: # The trailing asterisk is for handling the .exe extension on Windows. - - build/install/bin/complex_query_bm* - - build/install/bin/query_bm* - - build/install/bin/point_query_bm* - - build/install/bin/plan_cache_classic_bm* - - build/install/bin/distinct_scan_bm* - - build/install/bin/shapifying_bm* - - build/install/bin/profile_filter_bm* - - build/install/bin/percentile_algo_bm* - - build/install/bin/window_function_percentile_bm* - - build/install/bin/window_function_concat_arrays_bm* - - build/install/bin/canonical_query_bm* - - build/install/bin/query_planner_bm* - - build/install/bin/plan_cache_key_encoding_bm* - - build/install/bin/query_settings_lookup_bm* - - build/install/bin/rate_limiting_bm* - - build/install/bin/shapifying_bm* - - build/install/bin/sbe_builder_bm* + - bazel-bin/**/complex_query_bm* + - bazel-bin/**/query_bm* + - bazel-bin/**/point_query_bm* + - bazel-bin/**/plan_cache_classic_bm* + - bazel-bin/**/distinct_scan_bm* + - bazel-bin/**/shapifying_bm* + - bazel-bin/**/profile_filter_bm* + - bazel-bin/**/percentile_algo_bm* + - bazel-bin/**/window_function_percentile_bm* + - bazel-bin/**/window_function_concat_arrays_bm* + - bazel-bin/**/canonical_query_bm* + - bazel-bin/**/query_planner_bm* + - bazel-bin/**/plan_cache_key_encoding_bm* + - bazel-bin/**/query_settings_lookup_bm* + - bazel-bin/**/rate_limiting_bm* + - bazel-bin/**/shapifying_bm* + - bazel-bin/**/sbe_builder_bm* executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_replication.yml b/buildscripts/resmokeconfig/suites/benchmarks_replication.yml index 57d9ba5ceff..cb3d5d72f53 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_replication.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_replication.yml @@ -1,7 +1,7 @@ test_kind: benchmark_test selector: - root: build/repl_bm.txt + root: bazel-bin/install/install-repl_bm_test_list.txt executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_second_half.yml b/buildscripts/resmokeconfig/suites/benchmarks_second_half.yml index a36d5f76044..bddba358424 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_second_half.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_second_half.yml @@ -1,7 +1,7 @@ test_kind: benchmark_test selector: - root: build/benchmarks.txt + root: bazel-bin/install/install-second_half_bm_test_list.txt # Please always add the glob pattern for the canary tests as part of "include_files" section when # defining a new test suite for your benchmarks. Doing so ensures they'll still run as part of the # Evergreen task associated with the new test suite. @@ -11,42 +11,42 @@ selector: exclude_files: # Exclude everything from benchmarks_first_half.yml: - - build/install/bin/first_half_bm* - - build/install/bin/lock_manager_bm* - - build/install/bin/path_lower_bm* - - build/install/bin/abt_lower_bm* - - build/install/bin/string_bm* - - build/install/bin/stacktrace_bm* - - build/install/bin/placement_history_bm* - - build/install/bin/system_resource_canary_bm* - - build/install/bin/future_bm* - - build/install/bin/key_gen_bm* - - build/install/bin/histogram_bm* - - build/install/bin/document_source_group_bm* + - bazel-bin/**/first_half_bm* + - bazel-bin/**/lock_manager_bm* + - bazel-bin/**/path_lower_bm* + - bazel-bin/**/abt_lower_bm* + - bazel-bin/**/string_bm* + - bazel-bin/**/stacktrace_bm* + - bazel-bin/**/placement_history_bm* + - bazel-bin/**/system_resource_canary_bm* + - bazel-bin/**/future_bm* + - bazel-bin/**/key_gen_bm* + - bazel-bin/**/histogram_bm* + - bazel-bin/**/document_source_group_bm* # The trailing asterisk is for handling the .exe extension on Windows. # Hash table benchmark is really slow, don't run on evergreen - - build/install/bin/hash_table_bm* + - bazel-bin/**/hash_table_bm* # These benchmarks are being run as part of the benchmarks_query.yml - - build/install/bin/query_bm* - - build/install/bin/plan_cache_classic_bm* - - build/install/bin/point_query_bm* - - build/install/bin/distinct_scan_bm* + - bazel-bin/**/query_bm* + - bazel-bin/**/plan_cache_classic_bm* + - bazel-bin/**/point_query_bm* + - bazel-bin/**/distinct_scan_bm* # These benchmarks are being run as part of the benchmarks_expression*.yml - - build/install/bin/expression_bm* - - build/install/bin/sbe_expression_bm* + - bazel-bin/**/expression_bm* + - bazel-bin/**/sbe_expression_bm* # These benchmarks are being run as part of the benchmarks_streams.yml test suite. - - build/install/bin/streams_operator_dag_bm* - - build/install/bin/streams_window_operator_bm* + - bazel-bin/**/streams_operator_dag_bm* + - bazel-bin/**/streams_window_operator_bm* # These benchmarks are only run when modifying or upgrading the immutable library. - - build/install/bin/immutable_absl_comparison_bm* - - build/install/bin/immutable_std_comparison_bm* + - bazel-bin/**/immutable_absl_comparison_bm* + - bazel-bin/**/immutable_std_comparison_bm* # These benchmarks are being run as part of the benchmarks_util.yml test suite. - - build/install/bin/tracking_allocator_bm* + - bazel-bin/**/tracking_allocator_bm* # These benchmarks are being run as part of the benchmarks_sorter.yml test suite. - - build/install/bin/sorter_checksum_calculator_bm* + - bazel-bin/**/sorter_checksum_calculator_bm* # These benchmarks are being run as part of the benchmarks_audit.yml test suite. - - build/install/bin/audit_bm* + - bazel-bin/**/audit_bm* executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_sep.yml b/buildscripts/resmokeconfig/suites/benchmarks_sep.yml index 8337e40a1b3..2df371797b8 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_sep.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_sep.yml @@ -2,7 +2,7 @@ test_kind: benchmark_test selector: - root: build/sep_bm.txt + root: bazel-bin/install/install-sep_bm_test_list.txt executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_sharding.yml b/buildscripts/resmokeconfig/suites/benchmarks_sharding.yml index 2b83e80b64d..122866f9cf8 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_sharding.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_sharding.yml @@ -1,7 +1,7 @@ test_kind: benchmark_test selector: - root: build/sharding_bm.txt + root: bazel-bin/install/install-sharding_bm_test_list.txt executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_sorter.yml b/buildscripts/resmokeconfig/suites/benchmarks_sorter.yml index a4e4e88b166..d6f6ffb9e19 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_sorter.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_sorter.yml @@ -1,12 +1,12 @@ test_kind: benchmark_test selector: - root: build/benchmarks.txt + root: bazel-bin/install/install-mongo_benchmark-stripped_test_list.txt # Please always add the glob pattern for the canary tests as part of "include_files" section when # defining a new test suite for your benchmarks. Doing so ensures they'll still run as part of the # Evergreen task associated with the new test suite. include_files: - - build/install/bin/sorter_checksum_calculator_bm* + - bazel-bin/install/**/sorter_checksum_calculator_bm* executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_storage.yml b/buildscripts/resmokeconfig/suites/benchmarks_storage.yml index 14c06738a0d..2419065ca3e 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_storage.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_storage.yml @@ -1,7 +1,7 @@ test_kind: benchmark_test selector: - root: build/storage_bm.txt + root: bazel-bin/install/install-storage_bm_test_list.txt executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_streams.yml b/buildscripts/resmokeconfig/suites/benchmarks_streams.yml index 2a27ec07b70..eb940d24635 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_streams.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_streams.yml @@ -2,12 +2,12 @@ test_kind: benchmark_test selector: - root: build/benchmarks.txt + root: bazel-bin/install/install-mongo_benchmark-stripped_test_list.txt include_files: # The trailing asterisk is for handling the .exe extension on Windows. - - build/**/system_resource_canary_bm* - - build/install/bin/streams_operator_dag_bm* - - build/install/bin/streams_window_operator_bm* + - bazel-bin/**/system_resource_canary_bm* + - bazel-bin/**/streams_operator_dag_bm* + - bazel-bin/**/streams_window_operator_bm* executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/benchmarks_util.yml b/buildscripts/resmokeconfig/suites/benchmarks_util.yml index b5887dc33c7..348623e7617 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks_util.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks_util.yml @@ -1,12 +1,12 @@ test_kind: benchmark_test selector: - root: build/benchmarks.txt + root: bazel-bin/install/install-mongo_benchmark-stripped_test_list.txt # Please always add the glob pattern for the canary tests as part of "include_files" section when # defining a new test suite for your benchmarks. Doing so ensures they'll still run as part of the # Evergreen task associated with the new test suite. include_files: - - build/install/bin/tracking_allocator_bm* + - bazel-bin/**/tracking_allocator_bm* executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/integration_tests_replset.yml b/buildscripts/resmokeconfig/suites/integration_tests_replset.yml index 0e6bddb468f..ab141abd4b4 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_replset.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_replset.yml @@ -1,12 +1,11 @@ test_kind: cpp_integration_test selector: - root: build/integration_tests.txt + root: bazel-bin/install/install-mongo_integration_test_test_list.txt exclude_files: - - build/**/mongo/client/client_dbclient_connection_integration_test* # Needs connection to single host. - - build/install/bin/client_dbclient_connection_integration_test* # Needs connection to single host. - - build/**/network_interface_ssl_test* # Requires SSL - - build/**/grpc_transport_integration_test # Must be run with parameters in the integration_tests_standalone_grpc variant + - bazel-bin/**/client_dbclient_connection_integration_test* # Needs connection to single host. + - bazel-bin/**/network_interface_ssl_test* # Requires SSL + - bazel-bin/**/grpc_transport_integration_test # Must be run with parameters in the integration_tests_standalone_grpc variant executor: archive: diff --git a/buildscripts/resmokeconfig/suites/integration_tests_replset_ssl_auth.yml b/buildscripts/resmokeconfig/suites/integration_tests_replset_ssl_auth.yml index 0db10dd16c3..b4fbd12cd09 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_replset_ssl_auth.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_replset_ssl_auth.yml @@ -2,12 +2,11 @@ test_kind: cpp_integration_test selector: - root: build/integration_tests.txt + root: bazel-bin/install/install-mongo_integration_test_test_list.txt include_files: - - build/**/network_interface_ssl_test - - build/install/bin/network_interface_ssl_test + - bazel-bin/**/network_interface_ssl_test exclude_files: - - build/**/grpc_transport_integration_test # Must be run with parameters in the integration_tests_standalone_grpc variant + - bazel-bin/**/grpc_transport_integration_test # Must be run with parameters in the integration_tests_standalone_grpc variant config_variables: - &keyFile jstests/libs/authTestsKey diff --git a/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml b/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml index 10a53ee303d..3508e2fef2a 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml @@ -1,12 +1,11 @@ test_kind: cpp_integration_test selector: - root: build/integration_tests.txt + root: bazel-bin/install/install-mongo_integration_test_test_list.txt exclude_files: - - build/**/mongo/client/client_dbclient_connection_integration_test* # Needs sleep command - - build/install/bin/client_dbclient_connection_integration_test* # Needs sleep command - - build/**/network_interface_ssl_test* # Requires SSL - - build/**/grpc_transport_integration_test # Must be run with parameters in the integration_tests_standalone_grpc variant + - bazel-bin/**/client_dbclient_connection_integration_test* # Needs sleep command + - bazel-bin/**/network_interface_ssl_test* # Requires SSL + - bazel-bin/**/grpc_transport_integration_test # Must be run with parameters in the integration_tests_standalone_grpc variant executor: archive: diff --git a/buildscripts/resmokeconfig/suites/integration_tests_standalone.yml b/buildscripts/resmokeconfig/suites/integration_tests_standalone.yml index 85cba083857..95ee7e73960 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_standalone.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_standalone.yml @@ -1,12 +1,12 @@ test_kind: cpp_integration_test selector: - root: build/integration_tests.txt + root: bazel-bin/install/install-mongo_integration_test_test_list.txt exclude_files: # Tests replica set monitor. - - build/**/replica_set_monitor_integration_test* - - build/**/network_interface_ssl_test* # Requires SSL - - build/**/grpc_transport_integration_test # Must be run with parameters in the integration_tests_standalone_grpc variant + - bazel-bin/**/replica_set_monitor_integration_test* + - bazel-bin/**/network_interface_ssl_test* # Requires SSL + - bazel-bin/**/grpc_transport_integration_test # Must be run with parameters in the integration_tests_standalone_grpc variant executor: archive: diff --git a/buildscripts/resmokeconfig/suites/integration_tests_standalone_audit.yml b/buildscripts/resmokeconfig/suites/integration_tests_standalone_audit.yml index 07947a3115f..4fbe564e1ef 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_standalone_audit.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_standalone_audit.yml @@ -1,12 +1,12 @@ test_kind: cpp_integration_test selector: - root: build/integration_tests.txt + root: bazel-bin/install/install-mongo_integration_test_test_list.txt exclude_files: # Tests replica set monitor. - - build/**/replica_set_monitor_integration_test* - - build/**/network_interface_ssl_test* # Requires SSL - - build/**/grpc_transport_integration_test # Must be run with parameters in the integration_tests_standalone_grpc variant + - bazel-bin/**/replica_set_monitor_integration_test* + - bazel-bin/**/network_interface_ssl_test* # Requires SSL + - bazel-bin/**/grpc_transport_integration_test # Must be run with parameters in the integration_tests_standalone_grpc variant executor: archive: diff --git a/buildscripts/resmokeconfig/suites/integration_tests_standalone_grpc.yml b/buildscripts/resmokeconfig/suites/integration_tests_standalone_grpc.yml index 74a9b96c46d..9ccacb89d0f 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_standalone_grpc.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_standalone_grpc.yml @@ -1,10 +1,10 @@ test_kind: cpp_integration_test selector: - root: build/integration_tests.txt + root: bazel-bin/install/install-mongo_integration_test_test_list.txt include_files: - - build/**/grpc_transport_integration_test* - - build/**/executor_integration_test* + - bazel-bin/**/grpc_transport_integration_test* + - bazel-bin/**/executor_integration_test* executor: archive: diff --git a/buildscripts/resmokeconfig/suites/libfuzzer.yml b/buildscripts/resmokeconfig/suites/libfuzzer.yml index 84be527d5a4..33db77067cf 100644 --- a/buildscripts/resmokeconfig/suites/libfuzzer.yml +++ b/buildscripts/resmokeconfig/suites/libfuzzer.yml @@ -1,7 +1,7 @@ test_kind: cpp_libfuzzer_test selector: - root: build/libfuzzer_tests.txt + root: bazel-bin/install/install-mongo_fuzzer_test_test_list.txt executor: config: diff --git a/buildscripts/resmokeconfig/suites/pretty-printer-tests.yml b/buildscripts/resmokeconfig/suites/pretty-printer-tests.yml index fa101e6cfcc..64b1b3e5648 100644 --- a/buildscripts/resmokeconfig/suites/pretty-printer-tests.yml +++ b/buildscripts/resmokeconfig/suites/pretty-printer-tests.yml @@ -1,7 +1,7 @@ test_kind: pretty_printer_test selector: - root: build/pretty_printer_tests.txt + root: bazel-bin/install/install-dist-test-stripped_test_list.txt executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/unittests.yml b/buildscripts/resmokeconfig/suites/unittests.yml index 5bf28525966..19dead5a508 100644 --- a/buildscripts/resmokeconfig/suites/unittests.yml +++ b/buildscripts/resmokeconfig/suites/unittests.yml @@ -1,7 +1,7 @@ test_kind: cpp_unit_test selector: - root: build/unittests.txt + root: bazel-bin/install/install-mongo_unittest_test_list.txt executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/unittests_auth.yml b/buildscripts/resmokeconfig/suites/unittests_auth.yml index e85bbb75d69..3c97e28c63a 100644 --- a/buildscripts/resmokeconfig/suites/unittests_auth.yml +++ b/buildscripts/resmokeconfig/suites/unittests_auth.yml @@ -1,7 +1,7 @@ test_kind: cpp_unit_test selector: - root: build/unittests.txt + root: bazel-bin/install/install-mongo_unittest_test_list.txt include_files: - build/**/auth/* diff --git a/buildscripts/resmokeconfig/suites/unittests_client.yml b/buildscripts/resmokeconfig/suites/unittests_client.yml index 1d4498e9455..0a136681629 100644 --- a/buildscripts/resmokeconfig/suites/unittests_client.yml +++ b/buildscripts/resmokeconfig/suites/unittests_client.yml @@ -1,7 +1,7 @@ test_kind: cpp_unit_test selector: - root: build/unittests.txt + root: bazel-bin/install/install-mongo_unittest_test_list.txt include_files: - build/**/mongo/client/* diff --git a/buildscripts/resmokeconfig/suites/unittests_eighth_group.yml b/buildscripts/resmokeconfig/suites/unittests_eighth_group.yml index 8cc4d911f50..4aca583841a 100644 --- a/buildscripts/resmokeconfig/suites/unittests_eighth_group.yml +++ b/buildscripts/resmokeconfig/suites/unittests_eighth_group.yml @@ -1,7 +1,10 @@ test_kind: cpp_unit_test selector: - root: build/eighth_group_unittests.txt + root: bazel-bin/install/install-mongo_unittest_eighth_group_test_list.txt + + exclude_files: + - bazel-bin/**/s3_emit_operator_test* executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/unittests_fifth_group.yml b/buildscripts/resmokeconfig/suites/unittests_fifth_group.yml index db795c4a9ab..97460ea96c9 100644 --- a/buildscripts/resmokeconfig/suites/unittests_fifth_group.yml +++ b/buildscripts/resmokeconfig/suites/unittests_fifth_group.yml @@ -1,7 +1,7 @@ test_kind: cpp_unit_test selector: - root: build/fifth_group_unittests.txt + root: bazel-bin/install/install-mongo_unittest_fifth_group_test_list.txt executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/unittests_first_group.yml b/buildscripts/resmokeconfig/suites/unittests_first_group.yml index c0aef536ac4..9348e82e3ca 100644 --- a/buildscripts/resmokeconfig/suites/unittests_first_group.yml +++ b/buildscripts/resmokeconfig/suites/unittests_first_group.yml @@ -1,7 +1,7 @@ test_kind: cpp_unit_test selector: - root: build/first_group_unittests.txt + root: bazel-bin/install/install-mongo_unittest_first_group_test_list.txt executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/unittests_fourth_group.yml b/buildscripts/resmokeconfig/suites/unittests_fourth_group.yml index 575d5e4e9e0..4ab269e1178 100644 --- a/buildscripts/resmokeconfig/suites/unittests_fourth_group.yml +++ b/buildscripts/resmokeconfig/suites/unittests_fourth_group.yml @@ -1,7 +1,7 @@ test_kind: cpp_unit_test selector: - root: build/fourth_group_unittests.txt + root: bazel-bin/install/install-mongo_unittest_fourth_group_test_list.txt executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/unittests_query.yml b/buildscripts/resmokeconfig/suites/unittests_query.yml index e4a37edf462..9545b8847ea 100644 --- a/buildscripts/resmokeconfig/suites/unittests_query.yml +++ b/buildscripts/resmokeconfig/suites/unittests_query.yml @@ -1,7 +1,7 @@ test_kind: cpp_unit_test selector: - root: build/unittests.txt + root: bazel-bin/install/install-mongo_unittest_test_list.txt include_files: - build/**/pipeline/* - build/**/query/* diff --git a/buildscripts/resmokeconfig/suites/unittests_repl.yml b/buildscripts/resmokeconfig/suites/unittests_repl.yml index a9c5dc8ee2d..e2508ca6cbc 100644 --- a/buildscripts/resmokeconfig/suites/unittests_repl.yml +++ b/buildscripts/resmokeconfig/suites/unittests_repl.yml @@ -1,7 +1,7 @@ test_kind: cpp_unit_test selector: - root: build/unittests.txt + root: bazel-bin/install/install-mongo_unittest_test_list.txt include_files: - build/**/repl/* diff --git a/buildscripts/resmokeconfig/suites/unittests_second_group.yml b/buildscripts/resmokeconfig/suites/unittests_second_group.yml index 02071e090a2..9063f120c29 100644 --- a/buildscripts/resmokeconfig/suites/unittests_second_group.yml +++ b/buildscripts/resmokeconfig/suites/unittests_second_group.yml @@ -1,7 +1,10 @@ test_kind: cpp_unit_test selector: - root: build/second_group_unittests.txt + root: bazel-bin/install/install-mongo_unittest_second_group_test_list.txt + + exclude_files: + - bazel-bin/**/tracing_test* executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/unittests_seventh_group.yml b/buildscripts/resmokeconfig/suites/unittests_seventh_group.yml index e6bc3904f28..9b53a0c89e7 100644 --- a/buildscripts/resmokeconfig/suites/unittests_seventh_group.yml +++ b/buildscripts/resmokeconfig/suites/unittests_seventh_group.yml @@ -1,7 +1,7 @@ test_kind: cpp_unit_test selector: - root: build/seventh_group_unittests.txt + root: bazel-bin/install/install-mongo_unittest_seventh_group_test_list.txt executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/unittests_sharding.yml b/buildscripts/resmokeconfig/suites/unittests_sharding.yml index e550aaab91d..62de1fc9445 100644 --- a/buildscripts/resmokeconfig/suites/unittests_sharding.yml +++ b/buildscripts/resmokeconfig/suites/unittests_sharding.yml @@ -1,7 +1,7 @@ test_kind: cpp_unit_test selector: - root: build/unittests.txt + root: bazel-bin/install/install-mongo_unittest_test_list.txt include_files: - build/**/s/**/* diff --git a/buildscripts/resmokeconfig/suites/unittests_sixth_group.yml b/buildscripts/resmokeconfig/suites/unittests_sixth_group.yml index 22a1ff78b85..7d8ef1b3ed0 100644 --- a/buildscripts/resmokeconfig/suites/unittests_sixth_group.yml +++ b/buildscripts/resmokeconfig/suites/unittests_sixth_group.yml @@ -1,7 +1,7 @@ test_kind: cpp_unit_test selector: - root: build/sixth_group_unittests.txt + root: bazel-bin/install/install-mongo_unittest_sixth_group_test_list.txt executor: config: {} diff --git a/buildscripts/resmokeconfig/suites/unittests_third_group.yml b/buildscripts/resmokeconfig/suites/unittests_third_group.yml index 16879a2e348..f1914cff14d 100644 --- a/buildscripts/resmokeconfig/suites/unittests_third_group.yml +++ b/buildscripts/resmokeconfig/suites/unittests_third_group.yml @@ -1,7 +1,7 @@ test_kind: cpp_unit_test selector: - root: build/third_group_unittests.txt + root: bazel-bin/install/install-mongo_unittest_third_group_test_list.txt executor: config: {} diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 8c8392f6a8c..dcacfa83438 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -681,24 +681,24 @@ BENCHMARK_OUT_FORMAT = "json" ORDER_TESTS_BY_NAME = True # Default file names for externally generated lists of tests created during the build. -DEFAULT_BENCHMARK_TEST_LIST = "build/benchmarks.txt" -DEFAULT_UNIT_TEST_LIST = "build/unittests.txt" -DEFAULT_INTEGRATION_TEST_LIST = "build/integration_tests.txt" -DEFAULT_LIBFUZZER_TEST_LIST = "build/libfuzzer_tests.txt" -DEFAULT_PRETTY_PRINTER_TEST_LIST = "build/pretty_printer_tests.txt" +DEFAULT_BENCHMARK_TEST_LIST = "bazel-bin/install/install-mongo_benchmark-stripped_test_list.txt" +DEFAULT_UNIT_TEST_LIST = "bazel-bin/install/install-mongo_unittest_test_list.txt" +DEFAULT_INTEGRATION_TEST_LIST = "bazel-bin/install/install-mongo_integration_test_test_list.txt" +DEFAULT_LIBFUZZER_TEST_LIST = "bazel-bin/install/install-mongo_fuzzer_test_test_list.txt" +DEFAULT_PRETTY_PRINTER_TEST_LIST = "bazel-bin/install/install-dist-test-stripped_test_list.txt" SPLIT_UNITTESTS_LISTS = [ - f"build/{test_group}_group_unittests.txt" + f"bazel-bin/install/install-mongo_unittest_{test_group}_group_test_list.txt" for test_group in ["first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth"] ] BENCHMARK_SUITE_TEST_LISTS = [ - "build/repl_bm.txt", - "build/query_bm.txt", - "build/bsoncolumn_bm.txt", - "build/first_half_bm.txt", - "build/second_half_bm.txt", - "build/storage_bm.txt", - "build/sharding_bm.txt", - "build/sep_bm.txt", + "bazel-bin/install/install-repl_bm_test_list.txt", + "bazel-bin/install/install-query_bm_test_list.txt", + "bazel-bin/install/install-bsoncolumn_bm_test_list.txt", + "bazel-bin/install/install-first_half_bm_test_list.txt", + "bazel-bin/install/install-second_half_bm_test_list.txt", + "bazel-bin/install/install-storage_bm_test_list.txt", + "bazel-bin/install/install-sharding_bm_test_list.txt", + "bazel-bin/install/install-sep_bm_test_list.txt", ] # External files or executables, used as suite selectors, that are created during the build and # therefore might not be available when creating a test membership map. diff --git a/buildscripts/resmokelib/testing/hooks/metadata_consistency.py b/buildscripts/resmokelib/testing/hooks/metadata_consistency.py index 47ecf72850f..24efd85b6ea 100644 --- a/buildscripts/resmokelib/testing/hooks/metadata_consistency.py +++ b/buildscripts/resmokelib/testing/hooks/metadata_consistency.py @@ -23,9 +23,9 @@ class CheckMetadataConsistencyInBackground(jsfile.PerClusterDataConsistencyHook) # The 'CheckMetadataConsistency' hook relies on the 'isMaster' command to asses if the fixture cluster is sharded. SKIP_TESTS = [ # Skip tests that set a failPoint to make the 'isMaster' command unconditionally fail. - "build/install/bin/executor_integration_test", - "build/install/bin/rpc_integration_test", - "build/install/bin/asio_transport_integration_test", + "bazel-bin/install/bin/executor_integration_test", + "bazel-bin/install/bin/rpc_integration_test", + "bazel-bin/install/bin/asio_transport_integration_test", # Skip tests that update the internalDocumentSourceGroupMaxMemoryBytes parameter and make # checkMetadataConsistency fail with QueryExceededMemoryLimitNoDiskUseAllowed error. "jstests/aggregation/sources/unionWith/unionWith.js", diff --git a/etc/evergreen_yml_components/configuration.yml b/etc/evergreen_yml_components/configuration.yml index 90271a3f1b7..2274c79527f 100644 --- a/etc/evergreen_yml_components/configuration.yml +++ b/etc/evergreen_yml_components/configuration.yml @@ -59,8 +59,6 @@ parameters: - key: project_weekly_cron value: "0 4 * * 0" # Every week starting 0400 UTC Sunday description: "Cron schedule for nightly variants" - - key: bazel_scons_diff_targets - description: "Targets to run the bazel-scons diff on in this patch" - key: convert_bazel_headers_target description: "Target to generate bazel headers for" @@ -89,7 +87,6 @@ post: - func: "f_expansions_write" - func: "cleanup external auth OIDC resources" - func: "upload npm logs" - - func: "attach scons logs" - func: "attach local resmoke invocation" - func: "attach bazel invocation" - func: "create bazel test report" @@ -111,7 +108,6 @@ post: - func: "save mongo coredumps" - func: "generate hang analyzer tasks" - func: "save failed unittests" - - func: "save unstripped dbtest" - func: "save hang analyzer debugger files" - func: "save disk statistics" - func: "save system resource information" @@ -135,7 +131,6 @@ post: ${report_file|src/report.json} ${archive_file|src/archive.json} src/network_diagnostics.txt - - func: "umount shared scons directory" - func: "umount tmp directory" - func: "cleanup FUSE watchdog" - func: "cleanup environment" diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 7353e2d76c7..a3cdb0a2ebe 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -337,8 +337,8 @@ functions: - "evergreen/functions/binaries_extract.py" - "--tarball=mongo_benchmarks.tgz" - "--extraction-command=${decompress}" - - "--change-dir=build/install" - - "--move-output=build/install/benchmarks.txt:build/" + - "--change-dir=${extraction_change_dir}" + - "--move-output=dist-test/:bazel-bin/install" "get version expansions": &get_version_expansions command: s3.get @@ -527,11 +527,12 @@ functions: params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mongo-debugsymbols.${ext|tgz} + local_file: src/bazel-bin/dist-test-debug.${ext|tgz} remote_file: ${mongo_debugsymbols} bucket: mciuploads permissions: public-read content_type: ${content_type|application/gzip} + display_name: mongo-debugsymbols.${ext|tgz} "use WiredTiger develop": &use_wiredtiger_develop command: subprocess.exec @@ -541,14 +542,6 @@ functions: args: - "./src/evergreen/functions/wiredtiger_develop_use.sh" - "umount shared scons directory": - command: subprocess.exec - display_name: "shared scons directory umount" - params: - binary: bash - args: - - "./src/evergreen/functions/shared_scons_directory_umount.sh" - "umount tmp directory": - *f_expansions_write - command: subprocess.exec @@ -1327,91 +1320,6 @@ functions: - *check_run_tests_infrastructure_failure - *check_resmoke_failure - "scons splunk": &scons_splunk - command: subprocess.exec - display_name: "scons splunk" - params: - background: true - continue_on_err: true - binary: bash - args: - - "./src/evergreen/scons_splunk.sh" - - "scons compile": - - *get_version_expansions - - *apply_version_expansions - - *f_expansions_write - - *scons_splunk - - command: subprocess.exec - display_name: "scons compile sh" - type: test - params: - binary: bash - env: - evergreen_remote_exec: ${evergreen_remote_exec|off} - author_email: ${author_email} - args: - - "src/evergreen/scons_compile.sh" - - command: expansions.update - display_name: "expansions update install_dir" - params: - updates: - - key: install_dir - value: ${install_dir} - - *f_expansions_write - - command: s3.put - display_name: "attach bazel scons diff file" - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/build/merged_diff.md - remote_file: ${project}/${build_variant}/${revision}/bazel-scons-diff-${task_id}-${execution}.md - bucket: mciuploads - permissions: public-read - content_type: text/markdown - display_name: Bazel-SCons Diff File - - # This is only used to simulate local dev compilation. Do NOT use for anything else. - "scons raw compile": - - *get_version_expansions - - *apply_version_expansions - - *f_expansions_write - - *scons_splunk - - command: subprocess.exec - display_name: "scons raw compile sh" - type: test - params: - binary: bash - args: - - "src/evergreen/scons_raw_compile.sh" - - "bazel scons diff": - - *get_version_expansions - - *apply_version_expansions - - *f_expansions_write - - *scons_splunk - - command: subprocess.exec - display_name: "bazel scons diff" - type: test - params: - binary: bash - args: - - "src/evergreen/bazel_scons_diff.sh" - - "attach bazel scons diff file": - command: s3.put - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/build/merged_diff.md - remote_file: ${project}/${build_variant}/${revision}/bazel-scons-diff-${task_id}-${execution}.md - bucket: mciuploads - permissions: public-read - content_type: text/plain - display_name: Bazel-SCons Diff File - "verify build output present": # Args: # $output (string) - relpath to the file-to-be-checked @@ -1589,19 +1497,6 @@ functions: bucket: mciuploads extract_to: src - "generate compile expansions": &generate_compile_expansions - command: subprocess.exec - display_name: "compile expansions generate sh" - params: - binary: bash - args: - - "src/evergreen/functions/compile_expansions_generate.sh" - - "apply compile expansions": &apply_compile_expansions - command: expansions.update - params: - file: src/compile_expansions.yml - "do jepsen setup": - *f_expansions_write - command: github.generate_token @@ -2446,36 +2341,6 @@ functions: - *tar_failed_unittests - *archive_failed_unittests - "archive dbtest": &archive_dbtest - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: dbtest-binary.tgz - remote_file: ${project}/${build_variant}/${revision}/dbtest/dbtest-${build_id}-${task_name}-${execution}.tgz - bucket: mciuploads - permissions: public-read - content_type: application/tar - display_name: dbtest binary - Execution ${execution} - optional: true - - "archive dbtest debugsymbols": &archive_dbtest_debug - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: dbtest-debugsymbols.tgz - remote_file: ${project}/${build_variant}/${revision}/dbtest/dbtest-${build_id}-${task_name}-${execution}.tgz - bucket: mciuploads - permissions: public-read - content_type: application/tar - display_name: dbtest debugsymbols - optional: true - - "save unstripped dbtest": - - *archive_dbtest - - *archive_dbtest_debug - ### Process & archive artifacts from hung processes ### "run hang analyzer": - *f_expansions_write @@ -2610,50 +2475,6 @@ functions: display_name: Bazel JVM dump ### Attach report & artifacts ### - "attach scons config logs": &attach_scons_config_logs - command: s3.put - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/build/scons/config.log - remote_file: ${project}/${build_variant}/${revision}/artifacts/scons-config.log.${build_id}-${task_name}-${execution} - bucket: mciuploads - permissions: public-read - content_type: text/plain - display_name: SCons configure log - - "attach scons cache logs": &attach_scons_cache_logs - command: s3.put - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/scons_cache.log - content_type: text/plain - remote_file: ${project}/${build_variant}/${revision}/artifacts/scons-cache.log.${build_id}-${task_name}.${execution} - bucket: mciuploads - permissions: public-read - display_name: SCons cache debug log - - "attach scons stdout logs": &attach_scons_stdout_logs - command: s3.put - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/scons_stdout.log - content_type: text/plain - remote_file: ${project}/${build_variant}/${revision}/artifacts/scons-stdout.log.${build_id}-${task_name}.${execution} - bucket: mciuploads - permissions: public-read - display_name: SCons stdout log - - "attach scons logs": - - *attach_scons_config_logs - - *attach_scons_cache_logs - - *attach_scons_stdout_logs - "create bazel test report": command: subprocess.exec params: @@ -2990,7 +2811,7 @@ functions: - *process_code_coverage_data - *upload_code_coverage_report - "do scons setup": + "do bazel setup": - command: manifest.load - *git_get_shallow_project - *f_expansions_write @@ -3012,10 +2833,6 @@ functions: - *f_expansions_write - *set_up_win_mount_script - *f_expansions_write - - *generate_compile_expansions - - *f_expansions_write - - *apply_compile_expansions - - *f_expansions_write - *set_task_expansion_macros - *f_expansions_write - *get_engflow_key diff --git a/etc/evergreen_yml_components/tasks/compile_tasks.yml b/etc/evergreen_yml_components/tasks/compile_tasks.yml index de367fad0ab..9db869a3d03 100644 --- a/etc/evergreen_yml_components/tasks/compile_tasks.yml +++ b/etc/evergreen_yml_components/tasks/compile_tasks.yml @@ -33,8 +33,6 @@ variables: max_hosts: 1 tasks: [] setup_task: - - func: "f_expansions_write" - - func: "apply compile expansions" - func: "f_expansions_write" - func: "set task expansion macros" - func: "f_expansions_write" @@ -42,7 +40,6 @@ variables: - func: "fetch pgo profile" teardown_task: - func: "f_expansions_write" - - func: "attach scons logs" - func: "attach report" - func: "attach artifacts" - func: "attach local resmoke invocation" @@ -52,7 +49,6 @@ variables: - func: "save mongo coredumps" - func: "generate hang analyzer tasks" - func: "save failed unittests" - - func: "save unstripped dbtest" - func: "save bazel headers" - func: "save bazel jvm dump" - func: "save hang analyzer debugger files" @@ -99,11 +95,8 @@ variables: - func: "f_expansions_write" - func: "set up win mount script" - func: "f_expansions_write" - - func: "generate compile expansions" - - func: "f_expansions_write" teardown_group: - func: "f_expansions_write" - - func: "umount shared scons directory" - func: "cleanup environment" timeout: - func: "f_expansions_write" @@ -163,50 +156,6 @@ variables: # Tasks # ################################################ tasks: - ## compile - build bazel targets ## - ## The bazel migration is in progress & this - ## task ensures we can build with bazel. - - name: bazel_workstation_unittests - tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version - commands: - - func: "get engflow creds" - - func: "scons compile" - vars: - targets: >- - install-all - archive-all - bazel_build_tags: --bazel-build-tag=all - task_compile_flags: >- - ICECC= - CCACHE= - --build-profile=opt - --ninja=disabled - --keep-going - - command: s3.put - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/all-runtime.tgz - remote_file: ${all_mongo_binaries} - bucket: mciuploads - permissions: public-read - content_type: application/gzip - display_name: Binaries - - func: "scons compile" - vars: - targets: >- - prove-unittests - task_compile_flags: >- - ICECC= - CCACHE= - --build-profile=opt - --ninja=disabled - --keep-going - # TODO(SERVER-82195): simplify this to avoid duplication of parameters in other tests. - name: compile_dist_test_quick tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"] @@ -214,20 +163,15 @@ tasks: - name: version_expansions_gen variant: generate-tasks-for-version commands: - - func: "scons compile" + - func: "bazel compile" vars: targets: >- - build/opt/mongo/db/commands/libfsync_locked.a - build/opt/mongo/db/commands/libtest_commands_enabled.a - task_compile_flags: >- - ICECC= - --build-profile=opt - --ninja=disabled - --link-model=static - --modules= + //src/mongo/db/commands:fsync_locked_with_debug + bazel_args: >- + --config=evg - func: "verify build output present" vars: - output: build/opt/mongo/db/commands/libfsync_locked.a + output: bazel-bin/src/mongo/db/commands/libfsync_locked_with_debug.lo - name: run_bazel_program tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"] @@ -250,7 +194,7 @@ tasks: - func: "bazel compile" vars: targets: archive-dist-test - args: --config=opt --separate_debug=True + bazel_args: --config=opt --separate_debug=True - func: "run tests" vars: suite: core @@ -274,7 +218,7 @@ tasks: targets: //src/... # Use --linkopt="-s" to dramatically reduce link time, since we only care about hydrating cache # on CppCompile as CppLink is performed locally and not cached remotely. - args: >- + bazel_args: >- --config=fastbuild --define GIT_COMMIT_HASH=nogitversion --output_groups=compilation_outputs @@ -296,7 +240,7 @@ tasks: vars: targets: //src/... # TODO(SERVER-99213): limit to --jobs 32 once fission is enabled by default in CI - args: >- + bazel_args: >- --config=opt --define GIT_COMMIT_HASH=nogitversion --output_groups=compilation_outputs @@ -319,7 +263,7 @@ tasks: targets: //src/... # Use --linkopt="-s" to dramatically reduce link time, since we only care about hydrating cache # on CppCompile as CppLink is performed locally and not cached remotely. - args: >- + bazel_args: >- --config=dbg --define GIT_COMMIT_HASH=nogitversion --output_groups=compilation_outputs @@ -342,7 +286,7 @@ tasks: targets: //src/... # Use --linkopt="-s" to dramatically reduce link time, since we only care about hydrating cache # on CppCompile as CppLink is performed locally and not cached remotely. - args: >- + bazel_args: >- --config=dbg_aubsan --define GIT_COMMIT_HASH=nogitversion --output_groups=compilation_outputs @@ -365,7 +309,7 @@ tasks: targets: //src/... # Use --linkopt="-s" to dramatically reduce link time, since we only care about hydrating cache # on CppCompile as CppLink is performed locally and not cached remotely. - args: >- + bazel_args: >- --config=dbg_tsan --define GIT_COMMIT_HASH=nogitversion --output_groups=compilation_outputs @@ -375,7 +319,7 @@ tasks: - <<: *clang_tidy_task_template name: run_bazel_clang_tidy commands: - - func: "do scons setup" + - func: "do bazel setup" - func: "bazel compile" vars: # Note that we only run clang-tidy on the Mongo-authored targets, not on third-party code. @@ -397,39 +341,26 @@ tasks: args: >- --compilation_mode=dbg - ## compile - build all scons targets except unittests ## - - name: compile_dist_test_half - tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version - commands: - - func: "scons compile" - vars: - targets: >- - compile_first_half_non_test_source - ${additional_compile_targets|} - task_compile_flags: >- - PREFIX=dist-test - - name: compile_upload_benchmarks tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] depends_on: - name: version_expansions_gen variant: generate-tasks-for-version commands: - - func: "scons compile" + - func: "bazel compile" vars: targets: >- - install-benchmarks - archive-benchmarks - bazel_build_tags: --bazel-build-tag=mongo_benchmark --bazel-build-tag=mongo_library --bazel-build-tag=-repl_bm --bazel-build-tag=-query_bm --bazel-build-tag=-bsoncolumn_bm --bazel-build-tag=-first_half_bm --bazel-build-tag=-second_half_bm --bazel-build-tag=-storage_bm --bazel-build-tag=-sharding_bm --bazel-build-tag=-sep_bm + archive-mongo_benchmark-stripped + bazel_args: >- + --config=evg + --include_autogenerated_targets=True + --build_tag_filters=-repl_bm,-query_bm,-bsoncolumn_bm,-first_half_bm,-second_half_bm,-storage_bm,-sharding_bm,-sep_bm compiling_for_test: true - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/benchmarks.${ext|tgz} + local_file: src/bazel-bin/mongo_benchmark-stripped.${ext|tgz} remote_file: ${mongo_benchmarks} bucket: mciuploads permissions: public-read @@ -443,116 +374,68 @@ tasks: depends_on: - name: compile_upload_benchmarks commands: - - func: "scons compile" + - func: "bazel compile" vars: targets: >- - install-benchmarks-debug - archive-benchmarks-debug + archive-mongo_benchmark-debug + bazel_args: >- + --config=evg + --include_autogenerated_targets=True + --build_tag_filters=-repl_bm,-query_bm,-bsoncolumn_bm,-first_half_bm,-second_half_bm,-storage_bm,-sharding_bm,-sep_bm compiling_for_test: true - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/benchmarks-debugsymbols.${ext|tgz} + local_file: src/bazel-bin/mongo_benchmark-debug.${ext|tgz} remote_file: ${mongo_benchmarks_debugsymbols} bucket: mciuploads permissions: public-read content_type: application/tar display_name: Benchmarks Debug - - name: libdeps_graph_linting - tags: - [ - "assigned_to_jira_team_devprod_build", - "development_critical_single_variant", - ] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version - commands: - - func: "do scons setup" - - func: "f_expansions_write" - - - func: "scons compile" - vars: - task_compile_flags: >- - --link-model=dynamic - --force-macos-dynamic-link - --build-tools=next - targets: generate-libdeps-graph - bazel_build_tags: --bazel-build-tag=all - - command: subprocess.exec - params: - binary: bash - args: - - "src/evergreen/libdeps_setup.sh" - - - command: subprocess.exec - params: - binary: bash - args: - - "src/evergreen/libdeps_run.sh" - - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/results.txt - remote_file: ${project}/${build_variant}/${revision}/artifacts/libdeps-results.txt.${build_id}-${task_name}.${execution} - bucket: mciuploads - permissions: public-read - content_type: text/plain - display_name: Libdeps Linter Results - - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/libdeps.graphml.gz - remote_file: ${project}/${build_variant}/${revision}/artifacts/libdeps.graphml.${build_id}-${task_name}.${execution}.gz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/gzip} - display_name: Libdeps Graph Data - - ## compile_all - build all scons targets ## + ## compile_all - build all bazel targets ## - name: compile_all_future_git_tag_multiversion tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] depends_on: - - name: compile_dist_test_future_git_tag_multiversion + - name: version_expansions_future_git_tag_multiversion_gen commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-all-meta - bazel_build_tags: --bazel-build-tag=all + targets: //src/... + bazel_args: >- + --config=evg compiling_for_test: true - name: compile_all_but_not_unittests tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] depends_on: - - name: compile_dist_test + - name: archive_dist_test commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-all-meta-but-not-unittests + targets: //src/... compiling_for_test: true # The executables generated by the install-all target are not used after successfully being # compiled. We therefore link them without debug symbols to compile them faster. - task_compile_flags: >- - ${compile_all_but_not_unittests_flags|BAZEL_FLAGS=--linkopt=-s} - skip_archive: on - bazel_build_tags: --bazel-build-tag=-mongo_unittest --bazel-build-tag=mongo_binary --bazel-build-tag=mongo_benchmark --bazel-build-tag=mongo_integration_test + bazel_args: >- + --config=evg + --build_tag_filters=-mongo_unittest,mongo_binary,mongo_benchmark,mongo_integration_test + --linkopt=-s + ${compile_all_but_not_unittests_flags|--linkopt=-s} ## compile_unittests ## - name: compile_unittests_future_git_tag_multiversion tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] depends_on: - - name: compile_dist_test_future_git_tag_multiversion + - name: version_expansions_future_git_tag_multiversion_gen commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-unittests install-unittests-debug - bazel_build_tags: --bazel-build-tag=mongo_unittest --bazel-build-tag=mongo_binary_unittest + targets: install-mongo_unittest + bazel_args: >- + --config=evg + --include_autogenerated_targets=True compiling_for_test: true - name: compile_libraries_for_unittests @@ -562,13 +445,15 @@ tasks: variant: generate-tasks-for-version exec_timeout_secs: 32400 # 9 hours commands: - - func: "scons compile" + - func: "bazel compile" vars: # Target the conf to just run the bazel part of the build - targets: \$BUILD_ROOT/scons/\$VARIANT_DIR/sconf_temp + targets: //src/... compiling_for_test: true task_compile_flags: ${unittest_library_compile_flags} - bazel_build_tags: --bazel-build-tag=mongo_library + bazel_args: >- + --config=evg + --build_tag_filters=mongo_library - name: compile_and_run_unittests_first_group tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] @@ -578,12 +463,14 @@ tasks: - name: compile_libraries_for_unittests exec_timeout_secs: 32400 # 9 hours commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-first_group_unittests install-first_group_unittests-debug + targets: install-mongo_unittest_first_group compiling_for_test: true task_compile_flags: ${unittest_compile_flags} - bazel_build_tags: --bazel-build-tag=mongo_unittest_first_group --bazel-build-tag=mongo_binary_unittest + bazel_args: >- + --config=evg + --include_autogenerated_targets=True - func: "f_expansions_write" - func: "run diskstats" - func: "f_expansions_write" @@ -592,7 +479,7 @@ tasks: - func: "run tests" vars: suite: unittests_first_group - install_dir: build/install/bin + install_dir: bazel-bin/install/bin exec_timeout_secs: 32400 # 9 hours - name: compile_and_run_unittests_second_group @@ -603,12 +490,14 @@ tasks: - name: compile_libraries_for_unittests exec_timeout_secs: 32400 # 9 hours commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-second_group_unittests install-second_group_unittests-debug + targets: install-mongo_unittest_second_group compiling_for_test: true task_compile_flags: ${unittest_compile_flags} - bazel_build_tags: --bazel-build-tag=mongo_unittest_second_group --bazel-build-tag=mongo_binary_unittest + bazel_args: >- + --config=evg + --include_autogenerated_targets=True - func: "f_expansions_write" - func: "run diskstats" - func: "f_expansions_write" @@ -617,7 +506,7 @@ tasks: - func: "run tests" vars: suite: unittests_second_group - install_dir: build/install/bin + install_dir: bazel-bin/install/bin exec_timeout_secs: 32400 # 9 hours - name: compile_and_run_unittests_third_group @@ -628,12 +517,14 @@ tasks: - name: compile_libraries_for_unittests exec_timeout_secs: 32400 # 9 hours commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-third_group_unittests install-third_group_unittests-debug + targets: install-mongo_unittest_third_group compiling_for_test: true task_compile_flags: ${unittest_compile_flags} - bazel_build_tags: --bazel-build-tag=mongo_unittest_third_group --bazel-build-tag=mongo_binary_unittest + bazel_args: >- + --config=evg + --include_autogenerated_targets=True - func: "f_expansions_write" - func: "run diskstats" - func: "f_expansions_write" @@ -642,7 +533,7 @@ tasks: - func: "run tests" vars: suite: unittests_third_group - install_dir: build/install/bin + install_dir: bazel-bin/install/bin exec_timeout_secs: 32400 # 9 hours - name: compile_and_run_unittests_fourth_group @@ -653,12 +544,14 @@ tasks: - name: compile_libraries_for_unittests exec_timeout_secs: 32400 # 9 hours commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-fourth_group_unittests install-fourth_group_unittests-debug + targets: install-mongo_unittest_fourth_group compiling_for_test: true task_compile_flags: ${unittest_compile_flags} - bazel_build_tags: --bazel-build-tag=mongo_unittest_fourth_group --bazel-build-tag=mongo_binary_unittest + bazel_args: >- + --config=evg + --include_autogenerated_targets=True - func: "f_expansions_write" - func: "run diskstats" - func: "f_expansions_write" @@ -667,7 +560,7 @@ tasks: - func: "run tests" vars: suite: unittests_fourth_group - install_dir: build/install/bin + install_dir: bazel-bin/install/bin exec_timeout_secs: 32400 # 9 hours - name: compile_and_run_unittests_fifth_group @@ -678,12 +571,14 @@ tasks: - name: compile_libraries_for_unittests exec_timeout_secs: 32400 # 9 hours commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-fifth_group_unittests install-fifth_group_unittests-debug + targets: install-mongo_unittest_fifth_group compiling_for_test: true task_compile_flags: ${unittest_compile_flags} - bazel_build_tags: --bazel-build-tag=mongo_unittest_fifth_group --bazel-build-tag=mongo_binary_unittest + bazel_args: >- + --config=evg + --include_autogenerated_targets=True - func: "f_expansions_write" - func: "run diskstats" - func: "f_expansions_write" @@ -692,7 +587,7 @@ tasks: - func: "run tests" vars: suite: unittests_fifth_group - install_dir: build/install/bin + install_dir: bazel-bin/install/bin exec_timeout_secs: 32400 # 9 hours - name: compile_and_run_unittests_sixth_group @@ -703,12 +598,14 @@ tasks: - name: compile_libraries_for_unittests exec_timeout_secs: 32400 # 9 hours commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-sixth_group_unittests install-sixth_group_unittests-debug + targets: install-mongo_unittest_sixth_group compiling_for_test: true task_compile_flags: ${unittest_compile_flags} - bazel_build_tags: --bazel-build-tag=mongo_unittest_sixth_group --bazel-build-tag=mongo_binary_unittest + bazel_args: >- + --config=evg + --include_autogenerated_targets=True - func: "f_expansions_write" - func: "run diskstats" - func: "f_expansions_write" @@ -717,7 +614,7 @@ tasks: - func: "run tests" vars: suite: unittests_sixth_group - install_dir: build/install/bin + install_dir: bazel-bin/install/bin exec_timeout_secs: 32400 # 9 hours - name: compile_and_run_unittests_seventh_group @@ -728,12 +625,14 @@ tasks: - name: compile_libraries_for_unittests exec_timeout_secs: 32400 # 9 hours commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-seventh_group_unittests install-seventh_group_unittests-debug + targets: install-mongo_unittest_seventh_group compiling_for_test: true task_compile_flags: ${unittest_compile_flags} - bazel_build_tags: --bazel-build-tag=mongo_unittest_seventh_group --bazel-build-tag=mongo_binary_unittest + bazel_args: >- + --config=evg + --include_autogenerated_targets=True - func: "f_expansions_write" - func: "run diskstats" - func: "f_expansions_write" @@ -742,7 +641,7 @@ tasks: - func: "run tests" vars: suite: unittests_seventh_group - install_dir: build/install/bin + install_dir: bazel-bin/install/bin exec_timeout_secs: 32400 # 9 hours - name: compile_and_run_unittests_eighth_group @@ -753,12 +652,14 @@ tasks: - name: compile_libraries_for_unittests exec_timeout_secs: 32400 # 9 hours commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-eighth_group_unittests install-eighth_group_unittests-debug + targets: install-mongo_unittest_eighth_group compiling_for_test: true task_compile_flags: ${unittest_compile_flags} - bazel_build_tags: --bazel-build-tag=mongo_unittest_eighth_group --bazel-build-tag=mongo_binary_unittest + bazel_args: >- + --config=evg + --include_autogenerated_targets=True - func: "f_expansions_write" - func: "run diskstats" - func: "f_expansions_write" @@ -767,7 +668,7 @@ tasks: - func: "run tests" vars: suite: unittests_eighth_group - install_dir: build/install/bin + install_dir: bazel-bin/install/bin exec_timeout_secs: 32400 # 9 hours ## run_unittests ## @@ -784,7 +685,7 @@ tasks: - func: "run tests" vars: suite: unittests - install_dir: build/install/bin + install_dir: bazel-bin/install/bin ##compile_and_archive_libfuzzertests - build libfuzzertests ## - name: compile_and_archive_libfuzzertests @@ -793,17 +694,19 @@ tasks: - name: version_expansions_gen variant: generate-tasks-for-version commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: archive-fuzzertests + targets: archive-mongo_fuzzer_test compiling_for_test: true - bazel_build_tags: --bazel-build-tag=dist_test --bazel-build-tag=mongo_fuzzer_test + bazel_args: >- + --config=evg + --include_autogenerated_targets=True # Store the fuzzer executable, which we use to generate and run fuzzer inputs. - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: "src/fuzzertests-runtime.tgz" + local_file: "src/bazel-bin/mongo_fuzzer_test.${ext|tgz}" remote_file: "${project}/libfuzzer-tests/${build_variant}/${revision}/libfuzzer-tests.tgz" bucket: mciuploads permissions: private @@ -816,19 +719,18 @@ tasks: name: compile_dbtest tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] depends_on: - - name: compile_dist_test + - name: archive_dist_test commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-dbtest install-dbtest-debug - compiling_for_test: true - skip_archive: on - bazel_build_tags: --bazel-build-tag=dbtest + targets: install-dbtest + bazel_args: >- + --config=evg - <<: *compile_dbtest name: compile_dbtest_future_git_tag_multiversion depends_on: - - name: compile_dist_test_future_git_tag_multiversion + - name: version_expansions_future_git_tag_multiversion_gen ## run_dbtest ## - &run_dbtest @@ -845,7 +747,7 @@ tasks: - func: "run tests" vars: suite: dbtest - install_dir: build/install/bin + install_dir: bazel-bin/install/bin - <<: *run_dbtest name: run_dbtest_future_git_tag_multiversion @@ -858,12 +760,22 @@ tasks: depends_on: - name: compile_dbtest commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: archive-dbtest archive-dbtest-debug - compiling_for_test: true - skip_archive: on - bazel_build_tags: --bazel-build-tag=dbtest + targets: archive-dbtest + bazel_args: >- + --config=evg + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/bazel-bin/dbtest.${ext|tgz} + remote_file: ${project}/${build_variant}/${revision}/dbtest/dbtest-${build_id}-${task_name}-${execution}.tgz + bucket: mciuploads + permissions: public-read + content_type: application/tar + display_name: dbtest binary - Execution ${execution} + optional: true - <<: *archive_dbtest name: archive_dbtest_future_git_tag_multiversion @@ -873,14 +785,15 @@ tasks: - name: compile_integration_test tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] depends_on: - - name: compile_dist_test + - name: archive_dist_test commands: - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-integration-tests - skip_archive: on + targets: install-mongo_integration_test + bazel_args: >- + --config=evg + --include_autogenerated_targets=True task_compile_flags: ${integration_test_compile_flags} - bazel_build_tags: --bazel-build-tag=mongo_integration_test --bazel-build-tag=dist_test compiling_for_test: true - name: archive_jstestshell @@ -892,24 +805,23 @@ tasks: - func: "activate task" vars: task_to_activate: archive_jstestshell_debug - - func: "scons compile" + - func: "bazel compile" vars: &archive_jstestshell_vars - targets: archive-jstestshell - compiling_for_test: true - separate_debug: on - skip_archive: on - bazel_build_tags: --bazel-build-tag=jstestshell + targets: archive-mongo-stripped + bazel_args: >- + --config=evg task_compile_flags: >- - PREFIX=dist-test - --link-model=static - --dbg=on + --asan=False + --tsan=False + --ubsan=False + --linkstatic=True + --dbg=True --opt=on - --sanitize= - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mongodb-jstestshell.${ext|tgz} + local_file: src/bazel-bin/mongo-stripped.${ext|tgz} remote_file: ${mongo_jstestshell} bucket: mciuploads permissions: public-read @@ -922,16 +834,15 @@ tasks: - name: archive_jstestshell commands: - func: "f_expansions_write" - - func: "scons compile" + - func: "bazel compile" vars: <<: *archive_jstestshell_vars - targets: archive-jstestshell-debug - skip_archive: on + targets: archive-mongo-debug - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mongodb-jstestshell-debugsymbols.${ext|tgz} + local_file: src/bazel-bin/mongo-debug.${ext|tgz} remote_file: ${mongo_jstestshell_debugsymbols} bucket: mciuploads permissions: public-read @@ -952,29 +863,20 @@ tasks: - name: version_expansions_gen variant: generate-tasks-for-version commands: - - func: "do scons setup" + - func: "do bazel setup" - func: "f_expansions_write" - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-stitch-support install-stitch-support-debug install-stitch-support-dev - bazel_build_tags: --bazel-build-tag=stitch_support - task_compile_flags: >- - --link-model=dynamic-sdk - --ssl=off - --enable-http-client=off - --modules= - DESTDIR='$BUILD_ROOT/stitch-support-lib-$MONGO_VERSION' + targets: archive-stitch_support + bazel_args: >- + --config=evg_stitch + task_compile_flags: --linkstatic=True - func: "f_expansions_write" - - command: subprocess.exec - params: - binary: bash - args: - - "src/evergreen/stitch_support_create_lib_tar.sh" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: "src/build/stitch-support.tgz" + local_file: "src/bazel-bin/stitch_support.${ext|tgz}" remote_file: "${project}/stitch-support/${build_variant}/${revision}/stitch-support-${version}.tgz" bucket: mciuploads permissions: public-read @@ -995,18 +897,14 @@ tasks: depends_on: - name: stitch_support_create_lib commands: - - func: "do scons setup" + - func: "do bazel setup" - func: "f_expansions_write" - - func: "scons compile" + - func: "bazel compile" vars: - targets: install-stitch-support-test - bazel_build_tags: --bazel-build-tag=stitch_support_test + targets: install-stitch_support_test compiling_for_test: true - task_compile_flags: >- - --ssl=off - --enable-http-client=off - --modules= - DESTDIR='$BUILD_ROOT/stitch-support-lib-$MONGO_VERSION' + bazel_args: >- + --config=evg_stitch_test - func: "get and apply version expansions" - func: "f_expansions_write" - command: subprocess.exec @@ -1023,23 +921,17 @@ tasks: variant: generate-tasks-for-version commands: - func: "f_expansions_write" - - func: "scons compile" + - func: "bazel compile" vars: - targets: archive-mongo-crypt-dev archive-mongo-crypt-debug - bazel_build_tags: --bazel-build-tag=mongo_crypt - task_compile_flags: >- - --dbg=on - --opt=off - --allocator=system - --enterprise-features=fle - --js-engine=none - --link-model=dynamic-sdk - DESTDIR='$BUILD_ROOT/crypt-lib-$MONGO_VERSION' + targets: archive-mongo_crypt-debug archive-mongo_crypt-stripped + bazel_args: >- + --config=evg_crypt_dbg + task_compile_flags: --linkstatic=True - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: "src/mongo-crypt-dev.${ext|tgz}" + local_file: "src/bazel-bin/mongo_crypt-stripped.${ext|tgz}" remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shared_v1_dev-${version}.${ext|tgz}" bucket: mciuploads permissions: public-read @@ -1049,7 +941,7 @@ tasks: params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: "src/mongo-crypt-debug.${ext|tgz}" + local_file: "src/bazel-bin/mongo_crypt-debug.${ext|tgz}" remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shared_v1_debug-${version}.${ext|tgz}" bucket: mciuploads permissions: public-read @@ -1062,23 +954,17 @@ tasks: - name: crypt_create_debug_lib commands: - func: "f_expansions_write" - - func: "scons compile" + - func: "bazel compile" vars: - targets: archive-mongo-crypt-shlib-test + targets: archive-mongo_crypt_shlib_test-stripped compiling_for_test: true - bazel_build_tags: --bazel-build-tag=mongo_crypt - task_compile_flags: >- - --allocator=system - --enterprise-features=fle - --js-engine=none - --link-model=static - DESTDIR='$BUILD_ROOT/crypt-lib-$MONGO_VERSION' - RPATH='$$RPATH_ESCAPED_DOLLAR_ORIGIN/../lib' + bazel_args: >- + --config=evg_crypt_test - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: "src/mongo-crypt-shlib-test-runtime.${ext|tgz}" + local_file: "src/bazel-bin/mongo_crypt_shlib_test-stripped.${ext|tgz}" remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shlib_test-${version}.${ext|tgz}" bucket: mciuploads permissions: public-read @@ -1129,9 +1015,6 @@ tasks: - func: "set up venv" - func: "fetch packages" - func: "f_expansions_write" - - func: "generate compile expansions" - - func: "apply compile expansions" - - func: "f_expansions_write" - func: "set up remote credentials" vars: aws_key_remote: ${repo_aws_key} @@ -1194,9 +1077,6 @@ tasks: local_file: src/mongo-cryptd.tgz build_variants: *mongocryptd_variants - func: "f_expansions_write" - - func: "generate compile expansions" - - func: "apply compile expansions" - - func: "f_expansions_write" - func: "fetch dist debugsymbols" - func: "set up remote credentials" vars: @@ -1580,9 +1460,6 @@ tasks: - func: "f_expansions_write" - func: "get and apply version expansions" - func: "f_expansions_write" - - func: "generate compile expansions" - - func: "apply compile expansions" - - func: "f_expansions_write" - command: s3.get params: aws_key: ${aws_key} @@ -1591,9 +1468,6 @@ tasks: bucket: mciuploads local_file: src/mongo_crypt_shared_v1.${ext|tgz} - func: "f_expansions_write" - - func: "generate compile expansions" - - func: "apply compile expansions" - - func: "f_expansions_write" - func: "set up remote credentials" vars: aws_key_remote: ${repo_aws_key} @@ -1692,9 +1566,9 @@ tasks: ################################################ task_groups: - <<: *compile_task_group_template - name: compile_dist_test_TG + name: archive_dist_test_TG tasks: - - compile_dist_test + - archive_dist_test - <<: *compile_bazel_task_group_template name: compile_bazel_TG @@ -1707,11 +1581,6 @@ task_groups: - run_bazel_program - bazel_archive_and_run_dist_test - - <<: *compile_task_group_template - name: bazel_workstation_TG - tasks: - - bazel_workstation_unittests - # We want to run these sequentially to put less load on the cluster - <<: *compile_task_group_template name: hydrate_bazel_profile_TG @@ -1751,14 +1620,12 @@ task_groups: - <<: *compile_task_group_template name: compile_and_archive_dist_test_TG tasks: - - compile_dist_test - archive_dist_test - archive_dist_test_debug - <<: *compile_task_group_template name: compile_and_archive_dist_test_then_package_TG tasks: - - compile_dist_test - archive_dist_test - archive_dist_test_debug - package @@ -1773,7 +1640,6 @@ task_groups: - <<: *compile_task_group_template name: compile_test_serial_TG tasks: - - compile_dist_test - archive_dist_test - archive_dist_test_debug - compile_dbtest @@ -1789,7 +1655,6 @@ task_groups: - <<: *compile_task_group_template name: compile_test_serial_future_git_tag_multiversion_TG tasks: - - compile_dist_test_future_git_tag_multiversion - archive_dist_test_future_git_tag_multiversion - archive_dist_test_debug_future_git_tag_multiversion - compile_unittests_future_git_tag_multiversion @@ -1802,7 +1667,6 @@ task_groups: - <<: *compile_task_group_template name: compile_test_and_package_serial_TG tasks: - - compile_dist_test - archive_dist_test - archive_dist_test_debug # package task here to reduce peak diskspace SERVER-86662 @@ -1817,7 +1681,6 @@ task_groups: - <<: *compile_task_group_template name: compile_test_serial_no_unittests_TG tasks: - - compile_dist_test - archive_dist_test - archive_dist_test_debug - compile_dbtest @@ -1828,7 +1691,6 @@ task_groups: - <<: *compile_task_group_template name: compile_test_and_package_serial_no_unittests_TG tasks: - - compile_dist_test - archive_dist_test - archive_dist_test_debug # package task here to reduce peak diskspace SERVER-86662 @@ -1842,7 +1704,6 @@ task_groups: - <<: *compile_task_group_template name: compile_test_and_package_serial_lto_no_unittests_TG tasks: - - compile_dist_test - archive_dist_test - archive_dist_test_debug # package task here to reduce peak diskspace SERVER-86662 @@ -1858,7 +1719,6 @@ task_groups: - <<: *compile_task_group_template name: compile_test_benchmark_and_package_serial_no_unittests_TG tasks: - - compile_dist_test - archive_dist_test - archive_dist_test_debug # package task here to reduce peak diskspace SERVER-86662 @@ -1892,8 +1752,7 @@ task_groups: - compile_libraries_for_unittests # These `parallel` task groups are only appropriate for builders that - # use --link-model=dynamic, and have scons_cache_scope: shared and - # scons_cache_mode: all. Such builders are able to share all build + # use --linkstatic=False. Such builders are able to share all build # artifacts, and therefore will not repeatedly re-link the same # code. In that mode, it makes sense to run all of these tasks # concurrently, since they will share state across machines and can @@ -1903,7 +1762,6 @@ task_groups: - <<: *compile_task_group_template name: compile_test_parallel_core_stream_TG tasks: - - compile_dist_test - archive_dist_test - archive_dist_test_debug - compile_all_but_not_unittests @@ -1911,7 +1769,6 @@ task_groups: - <<: *compile_task_group_template name: compile_test_parallel_core_stream_and_pretty_printer_tests_TG tasks: - - compile_dist_test - archive_dist_test - archive_dist_test_debug - run_pretty_printer_tests @@ -1930,14 +1787,10 @@ task_groups: - name: crypt_build_debug_and_test setup_task: - - func: "f_expansions_write" - - func: "apply compile expansions" - func: "f_expansions_write" - func: "set task expansion macros" - func: "f_expansions_write" - func: "get engflow creds" - teardown_task: - - func: "attach scons logs" setup_group_can_fail_task: true setup_group: - command: manifest.load @@ -1951,11 +1804,6 @@ task_groups: - func: "get buildnumber" - func: "f_expansions_write" - func: "set up win mount script" - - func: "f_expansions_write" - - func: "generate compile expansions" - teardown_group: - - func: "f_expansions_write" - - func: "umount shared scons directory" max_hosts: 1 tasks: - "crypt_create_debug_lib" diff --git a/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml b/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml index 086677e0fe6..a5ad8d0431c 100644 --- a/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml +++ b/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml @@ -9,8 +9,6 @@ variables: max_hosts: 1 tasks: [] setup_task: - - func: "f_expansions_write" - - func: "apply compile expansions" - func: "f_expansions_write" - func: "set task expansion macros" - func: "f_expansions_write" @@ -18,7 +16,6 @@ variables: - func: "fetch pgo profile" teardown_task: - func: "f_expansions_write" - - func: "attach scons logs" - func: "attach report" - func: "attach artifacts" - func: "attach local resmoke invocation" @@ -28,7 +25,6 @@ variables: - func: "save mongo coredumps" - func: "generate hang analyzer tasks" - func: "save failed unittests" - - func: "save unstripped dbtest" - func: "save bazel headers" - func: "save bazel jvm dump" - func: "save hang analyzer debugger files" @@ -75,11 +71,8 @@ variables: - func: "f_expansions_write" - func: "set up win mount script" - func: "f_expansions_write" - - func: "generate compile expansions" - - func: "f_expansions_write" teardown_group: - func: "f_expansions_write" - - func: "umount shared scons directory" - func: "cleanup environment" timeout: - func: "f_expansions_write" @@ -90,7 +83,6 @@ task_groups: - <<: *compile_task_group_template name: compile_and_package_serial_no_unittests_TG tasks: - - compile_dist_test - archive_dist_test - archive_dist_test_debug @@ -115,49 +107,32 @@ tasks: - <<: *version_expansions_gen name: version_expansions_future_git_tag_multiversion_gen - ## compile - build all scons targets except unittests ## - - &compile_dist_test - name: compile_dist_test - tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version - commands: - - func: "scons compile" - vars: - targets: >- - install-dist-test - ${additional_compile_targets|} - bazel_build_tags: --bazel-build-tag=dist_test - skip_archive: on - task_compile_flags: >- - PREFIX=dist-test - - - <<: *compile_dist_test - name: compile_dist_test_future_git_tag_multiversion - depends_on: - - name: version_expansions_future_git_tag_multiversion_gen - # Sys-perf relies on the name of this task, please reach out before changing it. - &archive_dist_test name: archive_dist_test tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] depends_on: - - name: compile_dist_test + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "activate task" vars: task_to_activate: ${archive_dist_test_debug_task_name|archive_dist_test_debug} skip_for_patch_author: sys-perf-user - - func: "scons compile" + - func: "bazel compile" vars: targets: >- - archive-dist-test - bazel_build_tags: --bazel-build-tag=dist_test - skip_archive: on - task_compile_flags: >- - PREFIX=dist-test + archive-dist-test-stripped + ${additional_compile_targets|} + bazel_args: >- + --config=evg - func: "f_expansions_write" + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/functions/fix_tar.sh" + - "bazel-bin/dist-test-stripped.tgz" - command: subprocess.exec params: binary: bash @@ -165,7 +140,7 @@ tasks: args: - "src/evergreen/run_python_script.sh" - "evergreen/macos_notary.py" - - "mongodb-binaries.${ext|tgz}" + - "bazel-bin/dist-test-stripped.${ext|tgz}" - func: "BOLT" - command: s3.put @@ -173,7 +148,7 @@ tasks: optional: true aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mongodb-binaries.${ext|tgz} + local_file: src/bazel-bin/dist-test-stripped.${ext|tgz} remote_file: ${mongo_binaries} bucket: mciuploads permissions: public-read @@ -261,7 +236,7 @@ tasks: - <<: *archive_dist_test name: archive_dist_test_future_git_tag_multiversion depends_on: - - name: compile_dist_test_future_git_tag_multiversion + - name: version_expansions_future_git_tag_multiversion_gen - &archive_dist_test_debug name: archive_dist_test_debug @@ -269,14 +244,12 @@ tasks: depends_on: - name: archive_dist_test commands: - - func: "scons compile" + - func: "bazel compile" vars: targets: >- - archive-dist-test-debug - bazel_build_tags: --bazel-build-tag=dist_test - skip_archive: on - task_compile_flags: >- - PREFIX=dist-test + archive-dist-test-debug install-dist-test + bazel_args: >- + --config=evg - func: "upload debugsymbols" - func: "f_expansions_write" - command: subprocess.exec @@ -284,7 +257,7 @@ tasks: binary: bash args: - "./src/evergreen/record_mongodb_server_version.sh" - - "./build/install/dist-test/bin/mongod" + - "./bazel-bin/install/bin/mongod" - "./version_info_mongod.txt" - command: s3.put params: @@ -320,21 +293,14 @@ tasks: - name: version_expansions_gen variant: generate-tasks-for-version commands: - - func: "do scons setup" + - func: "do bazel setup" - func: "f_expansions_write" - - func: "scons compile" + - func: "bazel compile" vars: - targets: archive-mongo-crypt-dev - bazel_build_tags: --bazel-build-tag=mongo_crypt - task_compile_flags: >- - --allocator=system - --enterprise-features=fle - --js-engine=none - --link-model=dynamic-sdk - --enable-http-client=off - --ssl=off - ${crypt_task_compile_flags} - DESTDIR='$BUILD_ROOT/crypt-lib-$MONGO_VERSION' + targets: archive-mongo_crypt-stripped + bazel_args: >- + --config=evg_crypt + task_compile_flags: --linkstatic=True ${crypt_task_compile_flags} - command: subprocess.exec params: binary: bash @@ -344,7 +310,7 @@ tasks: params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: "src/mongo-crypt-dev.${ext|tgz}" + local_file: "src/bazel-bin/mongo_crypt-stripped.${ext|tgz}" remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shared_v1-${version}.${ext|tgz}" bucket: mciuploads permissions: public-read @@ -354,21 +320,25 @@ tasks: - name: package tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] depends_on: - - name: compile_dist_test + - name: archive_dist_test commands: - func: "f_expansions_write" - - func: "scons compile" + - func: "bazel compile" vars: targets: >- - archive-dist + archive-dist-stripped archive-dist-debug - archive-jstestshell - archive-jstestshell-debug + archive-mongo-stripped + archive-mongo-debug ${additional_package_targets|} - bazel_build_tags: --bazel-build-tag=dist_test - skip_archive: on - task_compile_flags: >- - --legacy-tarball + bazel_args: >- + --config=evg + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/functions/fix_tar.sh" + - "bazel-bin/dist-stripped.tgz" - command: subprocess.exec params: binary: bash @@ -376,7 +346,7 @@ tasks: args: - "src/evergreen/run_python_script.sh" - "evergreen/macos_notary.py" - - "mongodb-dist.${ext|tgz}" + - "bazel-bin/dist-stripped.${ext|tgz}" - func: "f_expansions_write" - command: subprocess.exec params: @@ -388,7 +358,7 @@ tasks: optional: true aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mongodb-dist.${ext|tgz} + local_file: src/bazel-bin/dist-stripped.${ext|tgz} remote_file: ${project}/${build_variant}/${revision}/dist/mongo-${build_id}.${ext|tgz} bucket: mciuploads permissions: public-read @@ -399,7 +369,7 @@ tasks: optional: true aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mongodb-dist-debugsymbols.${ext|tgz} + local_file: src/bazel-bin/dist-debug.${ext|tgz} remote_file: ${project}/${build_variant}/${revision}/dist/mongo-${build_id}-debugsymbols.${ext|tgz} bucket: mciuploads permissions: public-read @@ -409,7 +379,7 @@ tasks: params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mongodb-jstestshell.${ext|tgz} + local_file: src/bazel-bin/mongo-stripped.${ext|tgz} remote_file: ${mongo_jstestshell} bucket: mciuploads permissions: public-read @@ -419,7 +389,7 @@ tasks: params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mongodb-jstestshell-debugsymbols.${ext|tgz} + local_file: src/bazel-bin/mongo-debug.${ext|tgz} remote_file: ${mongo_jstestshell_debugsymbols} bucket: mciuploads permissions: public-read @@ -430,7 +400,7 @@ tasks: optional: true aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mongodb-cryptd.${ext|tgz} + local_file: src/bazel-bin/mongocryptd-stripped.${ext|tgz} remote_file: ${mongo_cryptd} bucket: mciuploads permissions: public-read @@ -441,7 +411,7 @@ tasks: optional: true aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mongodb-cryptd-debugsymbols.${ext|tgz} + local_file: src/bazel-bin/mongocryptd-debug.${ext|tgz} remote_file: ${mongo_cryptd_debugsymbols} bucket: mciuploads permissions: public-read @@ -454,7 +424,7 @@ tasks: source_dir: "src" include: - "repo/**" - - "./**.msi" + - "bazel-bin/src/mongo/installer/**.msi" - command: s3.put params: optional: true @@ -473,9 +443,9 @@ tasks: vars: files: >- packages.tgz - src/mongodb-cryptd-debugsymbols.${ext|tgz} - src/mongodb-cryptd.${ext|tgz} - src/mongodb-jstestshell-debugsymbols.${ext|tgz} - src/mongodb-jstestshell.${ext|tgz} - src/mongodb-dist-debugsymbols.${ext|tgz} - src/mongodb-dist.${ext|tgz} + src/bazel-bin/mongocryptd-debug.${ext|tgz} + src/bazel-bin/mongocryptd-stripped.${ext|tgz} + src/bazel-bin/mongo-debug.${ext|tgz} + src/bazel-bin/mongo-stripped.${ext|tgz} + src/bazel-bin/dist-debug.${ext|tgz} + src/bazel-bin/dist-stripped.${ext|tgz} diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index ba438d0d7bf..268de136847 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -436,7 +436,7 @@ tasks: "development_critical_single_variant", ] commands: - - func: "do scons setup" + - func: "do bazel setup" - func: "f_expansions_write" - command: subprocess.exec params: diff --git a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml index 7a6852bedec..9811e380fe9 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml @@ -93,7 +93,7 @@ tasks: - func: "run tests" vars: suite: pretty-printer-tests - install_dir: build/install/dist-test/bin + install_dir: bazel-bin/install/bin exec_timeout_secs: 1800 # 30 min timeout. - name: buildscripts_test @@ -180,6 +180,9 @@ tasks: ] commands: - func: "do setup" + vars: + extraction_change_dir: bazel-bin/install/ + decompress: tar --strip-components=1 -zxvf - func: "run tests" - <<: *task_template @@ -193,9 +196,6 @@ tasks: - name: archive_dist_test commands: - func: "do setup" - vars: - extraction_change_dir: build/install/ - move_outputs: "--move-output=build/install/dist-test/pretty_printer_tests.txt:build/" - func: "run tests" - <<: *gen_task_template @@ -207,6 +207,14 @@ tasks: "requires_compile_variant", "requires_execution_on_windows_patch_build", ] + depends_on: + - name: version_gen + variant: generate-tasks-for-version + # This is added because of EVG-18211. + # Without this we are adding extra dependencies on evergreen and it is causing strain + omit_generated_tasks: true + - name: archive_dist_test + - name: archive_dist_test_debug commands: - func: "generate resmoke tasks" vars: diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml index 945b2393f87..a972ad72a6a 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml @@ -126,15 +126,12 @@ variables: max_hosts: 1 tasks: [] setup_task: - - func: "f_expansions_write" - - func: "apply compile expansions" - func: "f_expansions_write" - func: "set task expansion macros" - func: "f_expansions_write" - func: "get engflow creds" teardown_task: - func: "f_expansions_write" - - func: "attach scons logs" - func: "attach report" - func: "attach artifacts" - func: "attach local resmoke invocation" @@ -144,7 +141,6 @@ variables: - func: "save mongo coredumps" - func: "generate hang analyzer tasks" - func: "save failed unittests" - - func: "save unstripped dbtest" - func: "save hang analyzer debugger files" - func: "save disk statistics" - func: "save system resource information" @@ -189,11 +185,8 @@ variables: - func: "f_expansions_write" - func: "set up win mount script" - func: "f_expansions_write" - - func: "generate compile expansions" - - func: "f_expansions_write" teardown_group: - func: "f_expansions_write" - - func: "umount shared scons directory" - func: "cleanup environment" timeout: - func: "f_expansions_write" @@ -232,36 +225,28 @@ tasks: "benchmarks", ] commands: - - func: "do benchmark setup" + - func: "do benchmark setup no fetch" - func: "get engflow creds" - - func: "scons compile" + - func: "bazel compile" vars: targets: >- - install-first_half_bm archive-first_half_bm - bazel_build_tags: --bazel-build-tag=first_half_bm + bazel_args: >- + --config=evg + --include_autogenerated_targets=True compiling_for_test: true - # scons compile resets install_dir - - command: expansions.update - display_name: "expansions fix install_dir" - params: - updates: - - key: install_dir - value: build/install/bin - - func: "f_expansions_write" - - func: "consolidate benchmark lists" - func: "run benchmark tests" vars: suite: benchmarks_first_half exec_timeout_secs: 28800 # 8 hour timeout. timeout_secs: 14400 # 4 hours resmoke_jobs_max: 1 - install_dir: build/install/bin + install_dir: bazel-bin/install/bin - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/first_half_bm.${ext|tgz} + local_file: src/bazel-bin/first_half_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/first_half_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads permissions: public-read @@ -277,35 +262,27 @@ tasks: "benchmarks", ] commands: - - func: "do benchmark setup" + - func: "do benchmark setup no fetch" - func: "get engflow creds" - - func: "scons compile" + - func: "bazel compile" vars: targets: >- - install-second_half_bm archive-second_half_bm - bazel_build_tags: --bazel-build-tag=second_half_bm + bazel_args: >- + --config=evg + --include_autogenerated_targets=True compiling_for_test: true - # scons compile resets install_dir - - command: expansions.update - display_name: "expansions fix install_dir" - params: - updates: - - key: install_dir - value: build/install/bin - - func: "f_expansions_write" - - func: "consolidate benchmark lists" - func: "run benchmark tests" vars: suite: benchmarks_second_half exec_timeout_secs: 28800 # 8 hour timeout. resmoke_jobs_max: 1 - install_dir: build/install/bin + install_dir: bazel-bin/install/bin - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/second_half_bm.${ext|tgz} + local_file: src/bazel-bin/second_half_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/second_half_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads permissions: public-read @@ -323,32 +300,25 @@ tasks: commands: - func: "do benchmark setup no fetch" - func: "get engflow creds" - - func: "scons compile" + - func: "bazel compile" vars: targets: >- - install-sep_bm archive-sep_bm - bazel_build_tags: --bazel-build-tag=sep_bm + bazel_args: >- + --config=evg + --include_autogenerated_targets=True compiling_for_test: true - # scons compile resets install_dir - - command: expansions.update - display_name: "expansions fix install_dir" - params: - updates: - - key: install_dir - value: build/install/bin - - func: "f_expansions_write" - func: "run benchmark tests" vars: suite: benchmarks_sep exec_timeout_secs: 28800 # 8 hour timeout. resmoke_jobs_max: 1 - install_dir: build/install/bin + install_dir: bazel-bin/install/bin - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/sep_bm.${ext|tgz} + local_file: src/bazel-bin/sep_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/sep_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads permissions: public-read @@ -717,7 +687,7 @@ tasks: - func: "do integration_tests setup" - func: "run tests" vars: - install_dir: build/install/bin + install_dir: bazel-bin/install/bin - <<: *task_template name: integration_tests_replset_ssl_auth @@ -733,7 +703,7 @@ tasks: - func: "do integration_tests setup" - func: "run tests" vars: - install_dir: build/install/bin + install_dir: bazel-bin/install/bin - <<: *task_template name: integration_tests_sharded @@ -750,7 +720,7 @@ tasks: - func: "do integration_tests setup" - func: "run tests" vars: - install_dir: build/install/bin + install_dir: bazel-bin/install/bin - <<: *task_template name: integration_tests_standalone @@ -767,7 +737,7 @@ tasks: - func: "do integration_tests setup" - func: "run tests" vars: - install_dir: build/install/bin + install_dir: bazel-bin/install/bin - <<: *task_template name: integration_tests_standalone_audit @@ -784,7 +754,7 @@ tasks: - func: "do integration_tests setup" - func: "run tests" vars: - install_dir: build/install/bin + install_dir: bazel-bin/install/bin - <<: *task_template name: integration_tests_standalone_grpc @@ -801,7 +771,7 @@ tasks: - func: "do integration_tests setup" - func: "run tests" vars: - install_dir: build/install/bin + install_dir: bazel-bin/install/bin - <<: *task_template name: libunwind_tests @@ -912,17 +882,17 @@ tasks: variant: generate-tasks-for-version - name: compile_libraries_for_unittests commands: - - func: "do scons setup" - - func: "scons compile" + - func: "do bazel setup" + - func: "bazel compile" vars: - targets: install-sdam-json-test + targets: install-sdam_json_test compiling_for_test: true - install_dir: build/install/bin - skip_archive: on - bazel_build_tags: --bazel-build-tag=sdam_json_test + bazel_args: >- + --config=evg - func: "run tests" vars: suite: sdam_json_test + install_dir: bazel-bin/install/bin - name: server_selection_json_test tags: @@ -937,15 +907,17 @@ tasks: variant: generate-tasks-for-version - name: compile_libraries_for_unittests commands: - - func: "do scons setup" - - func: "scons compile" + - func: "do bazel setup" + - func: "bazel compile" vars: - targets: install-server-selection-json-test + targets: install-server_selection_json_test compiling_for_test: true - install_dir: build/install/bin - skip_archive: on - bazel_build_tags: --bazel-build-tag=server_selection_json_test + bazel_args: >- + --config=evg - func: "run tests" + vars: + suite: server_selection_json_test + install_dir: bazel-bin/install/bin - <<: *task_template name: serverless @@ -1585,34 +1557,26 @@ tasks: "benchmarks", ] commands: - - func: "do benchmark setup" + - func: "do benchmark setup no fetch" - func: "get engflow creds" - - func: "scons compile" + - func: "bazel compile" vars: targets: >- - install-sharding_bm archive-sharding_bm - bazel_build_tags: --bazel-build-tag=sharding_bm + bazel_args: >- + --config=evg + --include_autogenerated_targets=True compiling_for_test: true - # scons compile resets install_dir - - command: expansions.update - display_name: "expansions fix install_dir" - params: - updates: - - key: install_dir - value: build/install/bin - - func: "f_expansions_write" - - func: "consolidate benchmark lists" - func: "run benchmark tests" vars: suite: benchmarks_sharding resmoke_jobs_max: 1 - install_dir: build/install/bin + install_dir: bazel-bin/install/bin - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/sharding_bm.${ext|tgz} + local_file: src/bazel-bin/sharding_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/sharding_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads permissions: public-read @@ -1628,34 +1592,26 @@ tasks: "benchmarks", ] commands: - - func: "do benchmark setup" + - func: "do benchmark setup no fetch" - func: "get engflow creds" - - func: "scons compile" + - func: "bazel compile" vars: targets: >- - install-bsoncolumn_bm archive-bsoncolumn_bm - bazel_build_tags: --bazel-build-tag=bsoncolumn_bm + bazel_args: >- + --config=evg + --include_autogenerated_targets=True compiling_for_test: true - # scons compile resets install_dir - - command: expansions.update - display_name: "expansions fix install_dir" - params: - updates: - - key: install_dir - value: build/install/bin - - func: "f_expansions_write" - - func: "consolidate benchmark lists" - func: "run benchmark tests" vars: suite: benchmarks_bsoncolumn resmoke_jobs_max: 1 - install_dir: build/install/bin + install_dir: bazel-bin/install/bin - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/bsoncolumn_bm.${ext|tgz} + local_file: src/bazel-bin/bsoncolumn_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/bsoncolumn_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads permissions: public-read @@ -3148,8 +3104,6 @@ task_groups: - <<: *compile_task_group_template name: small_compile_test_serial_no_unittests_TG tasks: - - compile_dist_test_half - - compile_dist_test - archive_dist_test - archive_dist_test_debug - compile_integration_test @@ -3171,8 +3125,6 @@ task_groups: - <<: *compile_task_group_template name: small_compile_test_and_package_serial_no_unittests_TG tasks: - - compile_dist_test_half - - compile_dist_test - archive_dist_test - archive_dist_test_debug - package diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml index e8497f4a4f6..27c85878629 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml @@ -115,34 +115,26 @@ tasks: name: benchmarks_replication tags: ["assigned_to_jira_team_server_repl", "experimental", "benchmarks"] commands: - - func: "do benchmark setup" + - func: "do benchmark setup no fetch" - func: "get engflow creds" - - func: "scons compile" + - func: "bazel compile" vars: targets: >- - install-repl_bm archive-repl_bm - bazel_build_tags: --bazel-build-tag=repl_bm + bazel_args: >- + --config=evg + --include_autogenerated_targets=True compiling_for_test: true - # scons compile resets install_dir - - command: expansions.update - display_name: "expansions fix install_dir" - params: - updates: - - key: install_dir - value: build/install/bin - - func: "f_expansions_write" - - func: "consolidate benchmark lists" - func: "run benchmark tests" vars: suite: benchmarks_replication - install_dir: build/install/bin + install_dir: bazel-bin/install/bin resmoke_jobs_max: 1 - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/repl_bm.${ext|tgz} + local_file: src/bazel-bin/repl_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/repl_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads permissions: public-read @@ -1707,34 +1699,26 @@ tasks: tags: ["assigned_to_jira_team_server_execution", "experimental", "benchmarks"] commands: - - func: "do benchmark setup" + - func: "do benchmark setup no fetch" - func: "get engflow creds" - - func: "scons compile" + - func: "bazel compile" vars: targets: >- - install-storage_bm archive-storage_bm - bazel_build_tags: --bazel-build-tag=storage_bm + bazel_args: >- + --config=evg + --include_autogenerated_targets=True compiling_for_test: true - # scons compile resets install_dir - - command: expansions.update - display_name: "expansions fix install_dir" - params: - updates: - - key: install_dir - value: build/install/bin - - func: "f_expansions_write" - - func: "consolidate benchmark lists" - func: "run benchmark tests" vars: suite: benchmarks_storage resmoke_jobs_max: 1 - install_dir: build/install/bin + install_dir: bazel-bin/install/bin - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/storage_bm.${ext|tgz} + local_file: src/bazel-bin/storage_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/storage_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads permissions: public-read diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml index e15f50f20b6..537b0c0e12f 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml @@ -1990,28 +1990,20 @@ tasks: "benchmarks", ] commands: - - func: "do benchmark setup" + - func: "do benchmark setup no fetch" - func: "get engflow creds" - - func: "scons compile" + - func: "bazel compile" vars: targets: >- - install-query_bm archive-query_bm - bazel_build_tags: --bazel-build-tag=query_bm + bazel_args: >- + --config=evg + --include_autogenerated_targets=True compiling_for_test: true - # scons compile resets install_dir - - command: expansions.update - display_name: "expansions fix install_dir" - params: - updates: - - key: install_dir - value: build/install/bin - - func: "f_expansions_write" - - func: "consolidate benchmark lists" - func: "run benchmark tests" vars: suite: benchmarks_query - install_dir: build/install/bin + install_dir: bazel-bin/install/bin resmoke_jobs_max: 1 exec_timeout_secs: 19800 # 5.5 hours timeout_secs: 19800 # 5.5 hours @@ -2019,7 +2011,7 @@ tasks: params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/query_bm.${ext|tgz} + local_file: src/bazel-bin/query_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/query_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads permissions: public-read diff --git a/etc/evergreen_yml_components/variants/amazon/test_dev.yml b/etc/evergreen_yml_components/variants/amazon/test_dev.yml index 9f11c567bf0..00ea268a938 100644 --- a/etc/evergreen_yml_components/variants/amazon/test_dev.yml +++ b/etc/evergreen_yml_components/variants/amazon/test_dev.yml @@ -16,19 +16,13 @@ variables: stepback: false - &linux-arm64-static-enterprise-compile-expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --link-model=static + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 unittest_library_compile_flags: >- - --link-model=dynamic + --linkstatic=False unittest_compile_flags: >- - --link-model=dynamic + --linkstatic=False # THIS HAS COPIES IN: # - etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml @@ -43,7 +37,6 @@ variables: repo_edition: enterprise large_distro_name: amazon2023-arm64-latest-large core_analyzer_distro_name: amazon2023-arm64-latest-xlarge - num_scons_link_jobs_available: 0.99 # THIS HAS COPIES IN: # - etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml @@ -74,8 +67,6 @@ variables: - &sys_perf_compile_expansions platform: linux project_dir: dsi - scons_cache_scope: shared - scons_cache_mode: all has_packages: false buildvariants: @@ -157,8 +148,6 @@ buildvariants: stepback: true expansions: <<: *amazon_linux2023_arm64_dynamic_expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false jstestfuzz_num_generated_files: 40 jstestfuzz_concurrent_num_files: 10 @@ -192,23 +181,14 @@ buildvariants: stepback: false expansions: <<: *linux_arm64_generic_expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false skip_symbolization: true evergreen_remote_exec: on - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --link-model=dynamic - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic - -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" + build_timeout_seconds: 1440 + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 + --linkstatic=False clang_tidy_toolchain: v5 - num_scons_unit_cc_jobs_available: 0.75 compile_variant: *commit-queue depends_on: [] tasks: @@ -246,8 +226,6 @@ buildvariants: stepback: true expansions: <<: *amazon_linux2023_arm64_dynamic_expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false jstestfuzz_num_generated_files: 20 jstestfuzz_concurrent_num_files: 5 @@ -300,13 +278,10 @@ buildvariants: expansions: <<: *sys_perf_compile_expansions compile_variant: amazon2023-x86-compile - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --release - --remote-exec-release - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 + --release=True + --config=local evergreen_remote_exec: on run_on: - amazon2023.3-xlarge @@ -321,13 +296,10 @@ buildvariants: expansions: <<: *sys_perf_compile_expansions compile_variant: amazon2023-arm64-compile - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --release - --remote-exec-release - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 + --release=True + --config=local evergreen_remote_exec: on run_on: - amazon2023.3-arm64-xlarge @@ -342,19 +314,16 @@ buildvariants: platform: linux project_dir: dsi has_packages: false - use_scons_cache: false pgo_profile_url: https://mciuploads.s3.amazonaws.com/dsi/perf-3-node-replSet.arm.aws.2024-05/b49fc707fb83a45727ad3f9f7cfa4c940a6805fe/sys_perf_perf_3_node_replSet.arm.aws.2024_05_ycsb.100read.2024_05_patch_b49fc707fb83a45727ad3f9f7cfa4c940a6805fe_66b035d75b27a40007f52c27_24_08_05_02_16_15/66b035d75b27a40007f52c27/logs/default.profdata.tgz compile_variant: amazon2023-arm64-compile-pgo - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --release - --remote-exec-release - --thin-lto - --pgo - --variables-files=etc/scons/mongodbtoolchain_stable_clang.vars - BAZEL_FLAGS="--keep_going" + # TODO(SERVER-102852): This is supposed to have --pgo or use --copt=-fprofile-use=./default.profdata, but bazel will have to support passing the default.profdata into the compiles which it does not currently + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 + --release=True + --config=local + --thin_lto=True + --compiler_type=clang + --keep_going run_on: - amazon2023.3-arm64-xlarge tasks: @@ -368,18 +337,14 @@ buildvariants: platform: linux project_dir: dsi has_packages: false - use_scons_cache: false bolt: true compile_variant: amazon2023-arm64-compile-BOLT - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --release - --remote-exec-release - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --bolt - BAZEL_FLAGS="--linkopt=-Wl,--emit-relocs" + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 + --release=True + --config=local + --bolt=True + --linkopt=-Wl,--emit-relocs run_on: - amazon2023.3-arm64-xlarge tasks: @@ -393,22 +358,10 @@ buildvariants: expansions: <<: *sys_perf_compile_expansions compile_variant: amazon2023-arm64-compile - use_scons_cache: true - compile_flags: >- - --separate-debug - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --release - --remote-exec-release - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --allocator=system - --enterprise-features=fle - --js-engine=none - --link-model=dynamic-sdk - --enable-http-client=off - --ssl=off - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 + --release=True + --config=local run_on: - amazon2023.3-arm64-xlarge tasks: @@ -421,21 +374,17 @@ buildvariants: run_on: - amazon2023-arm64-latest-small-m8g expansions: - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - BAZEL_FLAGS="--subcommands" - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 + --subcommands + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: >- --excludeWithAnyTags=incompatible_with_amazon_linux,requires_external_data_source --enableEnterpriseTests=off has_packages: false repo_edition: org - scons_cache_scope: shared large_distro_name: amazon2023-arm64-latest-small-m8g compile_variant: amazon2023-arm64 tasks: @@ -459,21 +408,16 @@ buildvariants: - amazon2023-arm64-latest-small-m8g expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" + --linkopt=-s test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,requires_v4_0,requires_external_data_source has_packages: false multiversion_platform: amazon2023 multiversion_edition: enterprise multiversion_architecture: aarch64 repo_edition: enterprise - scons_cache_scope: shared compile_variant: enterprise-amazon2023-arm64-grav4 tasks: - name: compile_test_and_package_serial_TG diff --git a/etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml index 10558a7011b..9b8045930c7 100644 --- a/etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml @@ -16,15 +16,10 @@ variables: stepback: false - &linux-arm64-dynamic-enterprise-compile-expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --link-model=dynamic + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 + --linkstatic=False # THIS HAS COPIES IN: # - etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml @@ -57,7 +52,6 @@ variables: repo_edition: enterprise large_distro_name: amazon2023-arm64-latest-large core_analyzer_distro_name: amazon2023-arm64-latest-xlarge - num_scons_link_jobs_available: 0.99 # THIS HAS COPIES IN: # - etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml @@ -86,14 +80,10 @@ buildvariants: run_on: - amazon2023-arm64-latest-large expansions: - scons_cache_scope: none - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --link-model=dynamic - BAZEL_FLAGS=--config=local + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 + --linkstatic=False + --config=local compile_variant: *amazon-linux2023-arm64-local-compile tasks: - name: compile_test_parallel_unittest_stream_TG @@ -125,11 +115,8 @@ buildvariants: stepback: false expansions: compile_variant: *rhel81-ppc64le-bazel-compile - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel81 - -j$(echo "$(grep -c processor /proc/cpuinfo)/2" | bc) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel81 tasks: - name: compile_bazel_TG @@ -144,12 +131,8 @@ buildvariants: stepback: false expansions: compile_variant: *rhel83-s390x-bazel-compile - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel83 - -j3 - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --linker=gold + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel83 tasks: - name: compile_bazel_TG @@ -164,8 +147,6 @@ buildvariants: stepback: true expansions: <<: *amazon_linux2023_arm64_dynamic_expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false jstestfuzz_num_generated_files: 40 jstestfuzz_concurrent_num_files: 10 @@ -198,8 +179,6 @@ buildvariants: cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. expansions: <<: *amazon_linux2023_arm64_dynamic_expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false jstestfuzz_num_generated_files: 40 jstestfuzz_concurrent_num_files: 10 @@ -302,14 +281,10 @@ buildvariants: run_on: - amazon2023.3-arm64-large expansions: - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --debug-symbols=minimal - separate_debug: off - scons_cache_scope: shared + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 + --dbg_level=1 + --separate_debug=False evergreen_remote_exec: on compile_variant: *al2023-arm64-sep-benchmark tasks: @@ -327,8 +302,6 @@ buildvariants: stepback: false expansions: <<: *amazon_linux2023_arm64_dynamic_expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false jstestfuzz_num_generated_files: 20 jstestfuzz_concurrent_num_files: 5 @@ -392,8 +365,6 @@ buildvariants: expansions: expansions: <<: *amazon_linux2023_arm64_dynamic_expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false jstestfuzz_num_generated_files: 20 jstestfuzz_concurrent_num_files: 5 @@ -447,8 +418,6 @@ buildvariants: expansions: expansions: <<: *amazon_linux2023_arm64_dynamic_expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false jstestfuzz_num_generated_files: 20 jstestfuzz_concurrent_num_files: 5 diff --git a/etc/evergreen_yml_components/variants/amazon/test_release.yml b/etc/evergreen_yml_components/variants/amazon/test_release.yml index 0075eda2207..a00fd6a4f10 100644 --- a/etc/evergreen_yml_components/variants/amazon/test_release.yml +++ b/etc/evergreen_yml_components/variants/amazon/test_release.yml @@ -18,14 +18,11 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-amazon2 - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2 + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s multiversion_platform: amazon2 multiversion_edition: targeted has_packages: true @@ -33,7 +30,6 @@ buildvariants: packager_arch: x86_64 packager_distro: amazon2 repo_edition: org - scons_cache_scope: shared large_distro_name: amazon2-latest-large compile_variant: amazon2 tasks: @@ -59,7 +55,7 @@ buildvariants: - amazon2-latest-small expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug test_flags: >- --excludeWithAnyTags=SERVER-34286,incompatible_with_amazon_linux,requires_external_data_source,requires_ldap_pool @@ -67,16 +63,10 @@ buildvariants: push_bucket: downloads.10gen.com push_name: linux push_arch: x86_64-enterprise-amazon2 - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" + --linkopt=-s multiversion_platform: amazon2 multiversion_edition: enterprise has_packages: true @@ -84,7 +74,6 @@ buildvariants: packager_arch: x86_64 packager_distro: amazon2 repo_edition: enterprise - scons_cache_scope: shared compile_variant: enterprise-amazon2 core_analyzer_distro_name: amazon2-latest-large large_distro_name: amazon2-latest-large @@ -117,14 +106,11 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: aarch64-amazon2 - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2 + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s multiversion_platform: amazon2 multiversion_edition: targeted has_packages: true @@ -132,7 +118,6 @@ buildvariants: packager_arch: aarch64 packager_distro: amazon2 repo_edition: org - scons_cache_scope: shared large_distro_name: amazon2-arm64-latest-large compile_variant: amazon2-arm64 tasks: @@ -158,7 +143,7 @@ buildvariants: - amazon2-arm64-latest-small expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug test_flags: >- --excludeWithAnyTags=SERVER-34286,incompatible_with_amazon_linux,requires_external_data_source,requires_ldap_pool @@ -166,16 +151,10 @@ buildvariants: push_bucket: downloads.10gen.com push_name: linux push_arch: aarch64-enterprise-amazon2 - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" + --linkopt=-s multiversion_platform: amazon2 multiversion_edition: enterprise has_packages: true @@ -183,7 +162,6 @@ buildvariants: packager_arch: aarch64 packager_distro: amazon2 repo_edition: enterprise - scons_cache_scope: shared compile_variant: enterprise-amazon2-arm64 core_analyzer_distro_name: amazon2-arm64-latest-large large_distro_name: amazon2-arm64-latest-large @@ -214,14 +192,11 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-amazon2023 - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: >- --excludeWithAnyTags=incompatible_with_amazon_linux,requires_external_data_source --enableEnterpriseTests=off @@ -230,7 +205,6 @@ buildvariants: packager_arch: x86_64 packager_distro: amazon2023 repo_edition: org - scons_cache_scope: shared large_distro_name: amazon2023.3-large compile_variant: amazon2023 tasks: @@ -256,19 +230,15 @@ buildvariants: run_on: - amazon2023.3-small expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug + additional_package_targets: archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: x86_64-enterprise-amazon2023 - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" + --linkopt=-s test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,requires_v4_0,requires_external_data_source has_packages: true multiversion_platform: amazon2023 @@ -278,7 +248,6 @@ buildvariants: packager_arch: x86_64 packager_distro: amazon2023 repo_edition: enterprise - scons_cache_scope: shared compile_variant: enterprise-amazon2023 tasks: - name: compile_test_and_package_serial_TG @@ -305,16 +274,12 @@ buildvariants: # run_on: # - amazon2023.3-small # expansions: - # additional_package_targets: archive-mongocryptd archive-mongocryptd-debug - # compile_flags: >- - # --ssl - # MONGO_DISTMOD=amazon2023 - # -j$(grep -c ^processor /proc/cpuinfo) - # --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + # additional_package_targets: archive-mongocryptd-stripped archive-mongocryptd-debug + # bazel_compile_flags: >- + # --define=MONGO_DISTMOD=amazon2023 # --lto # --linker=gold # --link-model=object - # crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" # test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,requires_v4_0,requires_external_data_source # has_packages: true # packager_script: packager_enterprise.py @@ -324,7 +289,6 @@ buildvariants: # multiversion_edition: enterprise # multiversion_architecture: x86_64 # repo_edition: enterprise - # scons_cache_scope: shared # compile_variant: enterprise-amazon2023-lto # large_distro_name: amazon2023.3-large # tasks: @@ -355,15 +319,12 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: aarch64-amazon2023 - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - BAZEL_FLAGS="--subcommands" - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 + --subcommands + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: >- --excludeWithAnyTags=incompatible_with_amazon_linux,requires_external_data_source --enableEnterpriseTests=off @@ -372,7 +333,6 @@ buildvariants: packager_arch: aarch64 packager_distro: amazon2023 repo_edition: org - scons_cache_scope: shared large_distro_name: amazon2023.3-arm64-large compile_variant: amazon2023-arm64 tasks: @@ -399,19 +359,15 @@ buildvariants: run_on: - amazon2023.3-arm64-small expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug + additional_package_targets: archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: aarch64-enterprise-amazon2023 - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" + --linkopt=-s test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,requires_v4_0,requires_external_data_source has_packages: true multiversion_platform: amazon2023 @@ -421,7 +377,6 @@ buildvariants: packager_arch: aarch64 packager_distro: amazon2023 repo_edition: enterprise - scons_cache_scope: shared compile_variant: enterprise-amazon2023-arm64 tasks: - name: compile_test_and_package_serial_TG @@ -449,16 +404,12 @@ buildvariants: # run_on: # - amazon2023.3-arm64-small # expansions: - # additional_package_targets: archive-mongocryptd archive-mongocryptd-debug - # compile_flags: >- - # --ssl - # MONGO_DISTMOD=amazon2023 - # -j$(grep -c ^processor /proc/cpuinfo) - # --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + # additional_package_targets: archive-mongocryptd-stripped archive-mongocryptd-debug + # bazel_compile_flags: >- + # --define=MONGO_DISTMOD=amazon2023 # --lto # --linker=gold # --link-model=object - # crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" # test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,requires_v4_0,requires_external_data_source # has_packages: true # multiversion_platform: amazon2023 @@ -468,7 +419,6 @@ buildvariants: # packager_arch: aarch64 # packager_distro: amazon2023 # repo_edition: enterprise - # scons_cache_scope: shared # compile_variant: enterprise-amazon2023-arm64-lto # large_distro_name: amazon2023.3-arm64-large # tasks: @@ -501,12 +451,9 @@ buildvariants: push_bucket: downloads.10gen.com push_name: linux push_arch: x86_64-enterprise-amazon2023-streams - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --streams-release-build + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 + --streams_release_build=True multiversion_platform: amazon2023 multiversion_edition: enterprise-streams has_packages: true @@ -514,7 +461,6 @@ buildvariants: packager_arch: x86_64 packager_distro: amazon2023 repo_edition: enterprise - scons_cache_scope: shared compile_variant: enterprise-amazon2023-streams large_distro_name: amazon2023-latest-large tasks: @@ -549,12 +495,9 @@ buildvariants: push_bucket: downloads.10gen.com push_name: linux push_arch: x86_64-enterprise-amazon2-streams - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --streams-release-build + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2 + --streams_release_build=True multiversion_platform: amazon2 multiversion_edition: enterprise-streams has_packages: true @@ -562,7 +505,6 @@ buildvariants: packager_arch: x86_64 packager_distro: amazon2 repo_edition: enterprise - scons_cache_scope: shared compile_variant: enterprise-amazon2-streams large_distro_name: amazon2-latest-large tasks: @@ -597,12 +539,9 @@ buildvariants: push_bucket: downloads.10gen.com push_name: linux push_arch: aarch64-enterprise-amazon2-streams - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --streams-release-build + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2 + --streams_release_build=True multiversion_platform: amazon2 multiversion_edition: enterprise-streams has_packages: true diff --git a/etc/evergreen_yml_components/variants/codecoverage/test_dev.yml b/etc/evergreen_yml_components/variants/codecoverage/test_dev.yml index 5b2da419f1c..effe11f62e4 100644 --- a/etc/evergreen_yml_components/variants/codecoverage/test_dev.yml +++ b/etc/evergreen_yml_components/variants/codecoverage/test_dev.yml @@ -10,30 +10,26 @@ buildvariants: stepback: false expansions: test_flags: --excludeWithAnyTags=resource_intensive,incompatible_with_gcov - compile_flags: >- + bazel_compile_flags: >- --allocator=system - --gcov --opt=off - --dbg=on - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --link-model=dynamic - BAZEL_FLAGS="${coverage_bazel_tags}" + --dbg=True + --define=MONGO_DISTMOD=amazon2023 + --linkstatic=False + --gcov=True + --collect_code_coverage + ${coverage_bazel_tags} large_distro_name: amazon2023.3-arm64-large resmoke_jobs_factor: 0.5 # Avoid starting too many mongod's # Mixing --cache and --gcov doesn't work correctly yet. See SERVER-11084 exec_timeout_secs: 32400 # 9 hour timeout timeout_secs: 18000 # 5 hour idle timeout - use_scons_cache: false gcov_tool: /opt/mongodbtoolchain/v4/bin/gcov # The gcov instrumentation saves the path the .gcno files were created in as the default path # for the .gcda files. In Evergreen the path will start with /data/mci/[Hashed ID]/src/... where # the hashed ID is unique per task run. GCOV_PREFIX_STRIP is the number of directory levels to # strip from the top of the default path before appending to the GCOV_PREFIX (if any). gcov_environment: GCOV_PREFIX=$(pwd)/.. GCOV_PREFIX_STRIP=3 - num_scons_link_jobs_available: 0.99 compile_variant: *AL2023-arm64-coverage tasks: # These are carefully and explicitly curated. Do not add more tests/tasks without consulting with DevProd. @@ -64,30 +60,25 @@ buildvariants: stepback: false expansions: test_flags: --excludeWithAnyTags=resource_intensive,incompatible_with_gcov - compile_flags: >- + bazel_compile_flags: >- --allocator=system - --gcov + --gcov=True --opt=off - --dbg=on - --ssl - MONGO_DISTMOD=rhel93 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --link-model=dynamic - BAZEL_FLAGS="${coverage_bazel_tags}" + --dbg=True + --define=MONGO_DISTMOD=rhel93 + --linkstatic=False + ${coverage_bazel_tags} large_distro_name: rhel93-medium resmoke_jobs_factor: 0.5 # Avoid starting too many mongod's # Mixing --cache and --gcov doesn't work correctly yet. See SERVER-11084 exec_timeout_secs: 32400 # 9 hour timeout timeout_secs: 18000 # 5 hour idle timeout - use_scons_cache: false gcov_tool: /opt/mongodbtoolchain/v4/bin/gcov # The gcov instrumentation saves the path the .gcno files were created in as the default path # for the .gcda files. In Evergreen the path will start with /data/mci/[Hashed ID]/src/... where # the hashed ID is unique per task run. GCOV_PREFIX_STRIP is the number of directory levels to # strip from the top of the default path before appending to the GCOV_PREFIX (if any). gcov_environment: GCOV_PREFIX=$(pwd)/.. GCOV_PREFIX_STRIP=3 - num_scons_link_jobs_available: 0.99 compile_variant: *rhel-93-64-bit-coverage tasks: # These are carefully and explicitly curated. Do not add more tests/tasks without consulting with DevProd. diff --git a/etc/evergreen_yml_components/variants/debian/test_release.yml b/etc/evergreen_yml_components/variants/debian/test_release.yml index 55cb369b508..64e597bbe9f 100644 --- a/etc/evergreen_yml_components/variants/debian/test_release.yml +++ b/etc/evergreen_yml_components/variants/debian/test_release.yml @@ -16,14 +16,11 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-debian12 - compile_flags: >- - --ssl - MONGO_DISTMOD=debian12 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=debian12 + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source,requires_updated_oscrypto --enableEnterpriseTests=off @@ -34,7 +31,6 @@ buildvariants: packager_arch: x86_64 packager_distro: debian12 repo_edition: org - scons_cache_scope: shared large_distro_name: debian12-large compile_variant: debian12 tasks: @@ -61,22 +57,17 @@ buildvariants: - debian12-small expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: x86_64-enterprise-debian12 - compile_flags: >- - --ssl MONGO_DISTMOD=debian12 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=debian12 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: --excludeWithAnyTags=requires_external_data_source,requires_updated_oscrypto - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" multiversion_platform: debian12 multiversion_edition: enterprise has_packages: true @@ -84,7 +75,6 @@ buildvariants: packager_arch: x86_64 packager_distro: debian12 repo_edition: enterprise - scons_cache_scope: shared large_distro_name: debian12-large compile_variant: enterprise-debian12-64 tasks: diff --git a/etc/evergreen_yml_components/variants/ibm/test_release_master_and_lts_branches_only.yml b/etc/evergreen_yml_components/variants/ibm/test_release_master_and_lts_branches_only.yml index b91001e31b0..9b7ee381945 100644 --- a/etc/evergreen_yml_components/variants/ibm/test_release_master_and_lts_branches_only.yml +++ b/etc/evergreen_yml_components/variants/ibm/test_release_master_and_lts_branches_only.yml @@ -15,18 +15,11 @@ buildvariants: stepback: false expansions: &enterprise-rhel-81-ppc64le-expansions-template additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug # We need to compensate for SMT8 setting the cpu count very high and lower the amount of parallelism down - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel81 - -j$(echo "$(grep -c processor /proc/cpuinfo)/2" | bc) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic - -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel81 resmoke_jobs_factor: 0.25 has_packages: true packager_script: packager_enterprise.py @@ -66,12 +59,9 @@ buildvariants: expansions: <<: *enterprise-rhel-81-ppc64le-expansions-template has_packages: false - compile_flags: >- - --link-model=dynamic - --ssl - MONGO_DISTMOD=rhel81 - -j$(echo "$(grep -c processor /proc/cpuinfo)/2" | bc) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --linkstatic=False + --define=MONGO_DISTMOD=rhel81 compile_variant: enterprise-rhel-81-ppc64le-shared tasks: - name: compile_test_serial_TG @@ -91,18 +81,11 @@ buildvariants: stepback: false expansions: &enterprise-rhel-9-ppc64le-expansions-template additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug # We need to compensate for SMT8 setting the cpu count very high and lower the amount of parallelism down - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel9 - -j$(echo "$(grep -c processor /proc/cpuinfo)/2" | bc) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic - -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel9 resmoke_jobs_factor: 0.25 has_packages: true packager_script: packager_enterprise.py @@ -142,12 +125,9 @@ buildvariants: expansions: <<: *enterprise-rhel-9-ppc64le-expansions-template has_packages: false - compile_flags: >- - --link-model=dynamic - --ssl - MONGO_DISTMOD=rhel9 - -j$(echo "$(grep -c processor /proc/cpuinfo)/2" | bc) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --linkstatic=False + --define=MONGO_DISTMOD=rhel9 compile_variant: enterprise-rhel-9-ppc64le-shared tasks: - name: compile_test_serial_TG @@ -167,19 +147,12 @@ buildvariants: stepback: false expansions: &enterprise-rhel-83-s390x-expansions-template additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug release_buid: true test_flags: --excludeWithAnyTags=incompatible_with_s390x - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel83 - -j3 - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --linker=gold - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel83 resmoke_jobs_max: 2 has_packages: true packager_script: packager_enterprise.py @@ -215,13 +188,9 @@ buildvariants: expansions: <<: *enterprise-rhel-83-s390x-expansions-template has_packages: false - compile_flags: >- - --link-model=dynamic - --ssl - MONGO_DISTMOD=rhel83 - -j$(echo "$(grep -c processor /proc/cpuinfo)/2" | bc) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --linker=gold + bazel_compile_flags: >- + --linkstatic=False + --define=MONGO_DISTMOD=rhel83 compile_variant: enterprise-rhel-83-s390x-shared exec_timeout_secs: 86400 # 24 hours exec timeout tasks: @@ -242,19 +211,12 @@ buildvariants: stepback: false expansions: &enterprise-rhel-9-s390x-expansions-template additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug release_buid: true test_flags: --excludeWithAnyTags=incompatible_with_s390x - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel9 - -j3 - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --linker=gold - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel9 resmoke_jobs_max: 2 has_packages: true packager_script: packager_enterprise.py @@ -290,13 +252,9 @@ buildvariants: expansions: <<: *enterprise-rhel-9-s390x-expansions-template has_packages: false - compile_flags: >- - --link-model=dynamic - --ssl - MONGO_DISTMOD=rhel9 - -j$(echo "$(grep -c processor /proc/cpuinfo)/2" | bc) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --linker=gold + bazel_compile_flags: >- + --linkstatic=False + --define=MONGO_DISTMOD=rhel9 compile_variant: enterprise-rhel-9-s390x-shared exec_timeout_secs: 86400 # 24 hours exec timeout tasks: diff --git a/etc/evergreen_yml_components/variants/macos/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/macos/test_dev_master_branch_only.yml index 346fa904c5f..e351e881da7 100644 --- a/etc/evergreen_yml_components/variants/macos/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/macos/test_dev_master_branch_only.yml @@ -13,14 +13,8 @@ buildvariants: expansions: compile_variant: *enterprise-macos-arm64 test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm - compile_env: DEVELOPER_DIR=/Applications/Xcode15.app - compile_flags: >- - --ssl - -j$(sysctl -n hw.logicalcpu) - --libc++ - --variables-files=etc/scons/xcode_macosx_arm.vars + bazel_compile_flags: --developer_dir=/Applications/Xcode15.app resmoke_jobs_max: 6 - num_scons_link_jobs_available: 0.99 tasks: - name: compile_test_serial_TG - name: compile_test_parallel_unittest_stream_TG diff --git a/etc/evergreen_yml_components/variants/macos/test_release.yml b/etc/evergreen_yml_components/variants/macos/test_release.yml index 8d09a92523d..dec93a372d9 100644 --- a/etc/evergreen_yml_components/variants/macos/test_release.yml +++ b/etc/evergreen_yml_components/variants/macos/test_release.yml @@ -19,13 +19,9 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: macos push_arch: arm64 - compile_env: DEVELOPER_DIR=/Applications/Xcode15.app - compile_flags: >- - --ssl - -j$(sysctl -n hw.logicalcpu) - --libc++ - --variables-files=etc/scons/xcode_macosx_arm.vars - --modules= + bazel_compile_flags: >- + --build_enterprise=False + --developer_dir=/Applications/Xcode15.app resmoke_jobs_max: 6 compile_variant: macos-arm64 tasks: @@ -43,18 +39,13 @@ buildvariants: expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm,requires_external_data_source additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug push_path: osx push_bucket: downloads.10gen.com push_name: macos push_arch: arm64-enterprise - compile_env: DEVELOPER_DIR=/Applications/Xcode15.app - compile_flags: >- - --ssl - -j$(sysctl -n hw.logicalcpu) - --libc++ - --variables-files=etc/scons/xcode_macosx_arm.vars + bazel_compile_flags: --developer_dir=/Applications/Xcode15.app resmoke_jobs_max: 6 compile_variant: enterprise-macos-arm64 tasks: @@ -77,13 +68,9 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: macos push_arch: x86_64 - compile_env: DEVELOPER_DIR=/Applications/Xcode15.app - compile_flags: >- - --ssl - -j$(sysctl -n hw.logicalcpu) - --libc++ - --variables-files=etc/scons/xcode_macosx.vars - --modules= + bazel_compile_flags: >- + --build_enterprise=False + --developer_dir=/Applications/Xcode15.app resmoke_jobs_max: 6 compile_variant: macos tasks: @@ -101,18 +88,13 @@ buildvariants: expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm,requires_external_data_source additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug push_path: osx push_bucket: downloads.10gen.com push_name: macos push_arch: x86_64-enterprise - compile_env: DEVELOPER_DIR=/Applications/Xcode15.app - compile_flags: >- - --ssl - -j$(sysctl -n hw.logicalcpu) - --libc++ - --variables-files=etc/scons/xcode_macosx.vars + bazel_compile_flags: --developer_dir=/Applications/Xcode15.app resmoke_jobs_max: 6 compile_variant: enterprise-macos tasks: diff --git a/etc/evergreen_yml_components/variants/mongot/test_dev.yml b/etc/evergreen_yml_components/variants/mongot/test_dev.yml index cf2e482839a..7eb9caa981b 100644 --- a/etc/evergreen_yml_components/variants/mongot/test_dev.yml +++ b/etc/evergreen_yml_components/variants/mongot/test_dev.yml @@ -11,11 +11,8 @@ buildvariants: - amazon2023.3-small expansions: &amazon-linux-2023-x86-mongot-integration-expansions build_mongot: true - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 test_flags: >- --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,requires_external_data_source --runAllFeatureFlagTests @@ -27,7 +24,6 @@ buildvariants: packager_arch: x86_64 packager_distro: amazon2023 repo_edition: enterprise - scons_cache_scope: shared compile_variant: amazon-linux-2023-x86-mongot-integration-cron-only core_analyzer_distro_name: amazon2023.3-large tasks: &amazon-linux-2023-x86-mongot-distro-tasks @@ -73,13 +69,10 @@ buildvariants: - amazon2023.3-arm64-small expansions: &amazon2023-arm64-mongot-integration-expansions build_mongot: true - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2023 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - BAZEL_FLAGS="--subcommands" - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2023 + --subcommands + --build_enterprise=False test_flags: >- --excludeWithAnyTags=incompatible_with_amazon_linux,requires_external_data_source --enableEnterpriseTests=off @@ -92,7 +85,6 @@ buildvariants: packager_arch: aarch64 packager_distro: amazon2023 repo_edition: enterprise - scons_cache_scope: shared compile_variant: amazon2023-arm64-mongot-integration-cron-only tasks: &amazon2023-arm64-mongot-distro-tasks - name: compile_and_archive_dist_test_TG @@ -138,13 +130,8 @@ buildvariants: stepback: true expansions: &amazon-linux2-arm64-mongot-integration-expansions build_mongot: true - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - scons_cache_scope: shared - scons_cache_mode: all + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2 has_packages: false target_resmoke_time: 10 max_sub_suites: 5 @@ -195,11 +182,8 @@ buildvariants: - amazon2-latest-large expansions: &amazon2-x86-mongot-integration-expansions build_mongot: true - compile_flags: >- - --ssl - MONGO_DISTMOD=amazon2 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=amazon2 test_flags: >- --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,requires_external_data_source --runAllFeatureFlagTests @@ -211,7 +195,6 @@ buildvariants: packager_arch: x86_64 packager_distro: amazon2 repo_edition: enterprise - scons_cache_scope: shared compile_variant: amazon2-x86-mongot-integration-cron-only tasks: &amazon2-x86-mongot-distro-tasks - name: compile_and_archive_dist_test_TG @@ -253,17 +236,12 @@ buildvariants: - rhel8.8-medium tags: [] expansions: - scons_cache_scope: shared - scons_cache_mode: all compile_variant: *enterprise-rhel-8-64-mongot-integration has_packages: false build_mongot: true - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --link-model=dynamic + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 + --linkstatic=False tasks: - name: compile_and_archive_dist_test_TG distros: diff --git a/etc/evergreen_yml_components/variants/rhel/test_dev.yml b/etc/evergreen_yml_components/variants/rhel/test_dev.yml index 685b4118067..b95ce542a09 100644 --- a/etc/evergreen_yml_components/variants/rhel/test_dev.yml +++ b/etc/evergreen_yml_components/variants/rhel/test_dev.yml @@ -20,15 +20,10 @@ variables: # - etc/evergreen_yml_components/variants/rhel/test_dev.yml # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &linux-x86-dynamic-enterprise-compile-expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --link-model=dynamic + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 + --linkstatic=False # THIS HAS COPIES IN: # - etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml @@ -70,7 +65,6 @@ variables: repo_edition: enterprise large_distro_name: rhel8.8-medium core_analyzer_distro_name: rhel8.8-xlarge - num_scons_link_jobs_available: 0.99 compile_variant: *linux_x86_dynamic_compile_variant_name # THIS HAS COPIES IN: @@ -81,8 +75,6 @@ variables: # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &enterprise-rhel-8-64-bit-dynamic-expansions <<: *linux_x86_generic_expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false jstestfuzz_num_generated_files: 40 jstestfuzz_concurrent_num_files: 10 diff --git a/etc/evergreen_yml_components/variants/rhel/test_dev_master_and_lts_branches_only.yml b/etc/evergreen_yml_components/variants/rhel/test_dev_master_and_lts_branches_only.yml index 4b58fa4ea2d..52128887409 100644 --- a/etc/evergreen_yml_components/variants/rhel/test_dev_master_and_lts_branches_only.yml +++ b/etc/evergreen_yml_components/variants/rhel/test_dev_master_and_lts_branches_only.yml @@ -45,7 +45,6 @@ variables: repo_edition: enterprise large_distro_name: rhel8.8-medium core_analyzer_distro_name: rhel8.8-xlarge - num_scons_link_jobs_available: 0.99 compile_variant: *linux_x86_dynamic_compile_variant_name buildvariants: @@ -57,20 +56,16 @@ buildvariants: - rhel8.8-small expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug test_flags: >- --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 multiversion_platform: rhel80 multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise - scons_cache_scope: shared large_distro_name: rhel8.8-large core_analyzer_distro_name: rhel8.8-xlarge compile_variant: enterprise-rhel-8-64-bit-inmem diff --git a/etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml index 9c2e229d0dc..2ecfdbee4d5 100644 --- a/etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml @@ -20,15 +20,9 @@ variables: # - etc/evergreen_yml_components/variants/rhel/test_dev.yml # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &linux-x86-dynamic-enterprise-compile-expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --link-model=dynamic + bazel_compile_flags: >- + --linkstatic=False # THIS HAS COPIES IN # - etc/evergreen_yml_components/variants/sanitizer/test_dev.yml @@ -52,8 +46,6 @@ variables: # - etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &generic_linux_compile_expansions # The most commonly used compile expansions. - scons_cache_scope: shared - scons_cache_mode: all has_packages: false # THIS HAS COPIES IN: @@ -96,7 +88,6 @@ variables: repo_edition: enterprise large_distro_name: rhel8.8-medium core_analyzer_distro_name: rhel8.8-xlarge - num_scons_link_jobs_available: 0.99 compile_variant: *linux_x86_dynamic_compile_variant_name # THIS HAS COPIES IN: @@ -107,8 +98,6 @@ variables: # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &enterprise-rhel-8-64-bit-dynamic-expansions <<: *linux_x86_generic_expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false jstestfuzz_num_generated_files: 40 jstestfuzz_concurrent_num_files: 10 @@ -126,20 +115,14 @@ variables: - rhel8.8-small expansions: &enterprise-rhel-8-64-bit-expansions-template additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: x86_64-enterprise-rhel8 - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 test_flags: --excludeWithAnyTags=requires_ldap_pool multiversion_platform: rhel80 multiversion_platform_50_or_later: rhel8 @@ -149,7 +132,6 @@ variables: packager_arch: x86_64 packager_distro: rhel88 repo_edition: enterprise - scons_cache_scope: shared core_analyzer_distro_name: rhel8.8-large tasks: - name: compile_test_and_package_serial_no_unittests_TG @@ -224,7 +206,6 @@ buildvariants: display_name: "* Linux x86 Stitch Enterprise Compile" tags: ["suggested"] expansions: - <<: *linux-x86-dynamic-enterprise-compile-expansions compile_variant: *linux-stitch-compile-suggested tasks: - name: .stitch @@ -234,11 +215,6 @@ buildvariants: display_name: "* Linux x86 Crypt Enterprise Compile" tags: ["suggested"] expansions: - <<: *linux-x86-dynamic-enterprise-compile-expansions - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic - -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" compile_variant: *linux-crypt-compile tasks: - name: .crypt @@ -252,12 +228,9 @@ buildvariants: stepback: false expansions: <<: *generic_linux_compile_expansions - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --link-model=dynamic + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 + --linkstatic=False compile_variant: *linux-x86-dynamic-grpc clang_tidy_toolchain: v5 large_distro_name: rhel8.8-xlarge @@ -284,7 +257,6 @@ buildvariants: - name: .concurrency !.auth !.embedded_router !concurrency_replication_causal_consistency_gen !concurrency_replication_multi_stmt_txn_gen !concurrency_replication_gen !concurrency_simultaneous_replication_gen !config_fuzzer_concurrency_replication_gen !config_fuzzer_concurrency_simultaneous_replication_gen - name: .grpc_misc_js - name: .lint - - name: libdeps_graph_linting distros: - rhel8.8-xlarge - name: .clang_tidy @@ -368,15 +340,10 @@ buildvariants: run_on: - rhel8.8-large expansions: - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --debug-symbols=minimal - separate_debug: off - scons_cache_scope: shared - num_scons_link_jobs_available: 0.5 + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 + --dbg_level=1 + --separate_debug=False compile_variant: *enterprise-rhel-8-benchmarks tasks: - name: compile_upload_benchmarks_TG @@ -421,8 +388,6 @@ buildvariants: tags: [] expansions: <<: *linux_x86_generic_expansions - scons_cache_scope: shared - scons_cache_mode: all resmoke_jobs_factor: 0.5 bv_future_git_tag: r100.0.0-9999 compile_variant: *linux-x86-dynamic-compile-future-tag-multiversion-latest @@ -455,18 +420,17 @@ buildvariants: expansions: <<: *enterprise-rhel-8-64-bit-expansions-template additional_package_targets: "" - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --enterprise-features=hot_backups,magic_restore,ldap,sasl,encryptdb - --link-model=dynamic + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 + --//bazel/config:enterprise_feature_all=False + --//bazel/config:enterprise_feature_hot_backups=True + --//bazel/config:enterprise_feature_magic_restore=True + --//bazel/config:enterprise_feature_ldap=True + --//bazel/config:enterprise_feature_sasl=True + --//bazel/config:enterprise_feature_encryptdb=True + --linkstatic=False compile_variant: *hot_backups-rhel-8-64-bit has_packages: false - scons_cache_mode: all - scons_cache_scope: shared - num_scons_link_jobs_available: 0.99 tasks: - name: compile_test_parallel_core_stream_TG distros: @@ -489,11 +453,9 @@ buildvariants: - rhel8.8-small expansions: <<: *linux_x86_generic_expansions - compile_flags: >- - MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --link-model=dynamic + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 + --linkstatic=False test_flags: >- --mongosSetParameters="{replicaSetMonitorProtocol: sdam}" --mongodSetParameters="{replicaSetMonitorProtocol: sdam}" @@ -573,12 +535,9 @@ buildvariants: - rhel8.8-small expansions: <<: *linux_x86_generic_expansions - scons_cache_scope: shared - compile_flags: >- - MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --link-model=dynamic + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 + --linkstatic=False multiversion_platform: rhel80 multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise diff --git a/etc/evergreen_yml_components/variants/rhel/test_release.yml b/etc/evergreen_yml_components/variants/rhel/test_release.yml index 7d4fbf06047..f4815be6ce4 100644 --- a/etc/evergreen_yml_components/variants/rhel/test_release.yml +++ b/etc/evergreen_yml_components/variants/rhel/test_release.yml @@ -13,13 +13,9 @@ buildvariants: - ubi8 expansions: resmoke_jobs_factor: 1 - disable_shared_scons_cache: true - compile_flags: >- - MONGO_DISTMOD=rhel88 - --opt=on - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 + --build_enterprise=False tooltags: "" build_mongoreplay: true test_flags: >- @@ -48,11 +44,9 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-rhel8 - compile_flags: >- - --ssl MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 + --build_enterprise=False test_flags: >- --excludeWithAnyTags=requires_external_data_source,requires_increased_memlock_limits --enableEnterpriseTests=off @@ -64,7 +58,6 @@ buildvariants: packager_arch: x86_64 packager_distro: rhel88 repo_edition: org - scons_cache_scope: shared large_distro_name: rhel8.8-large compile_variant: rhel88 tasks: @@ -92,22 +85,17 @@ buildvariants: - rhel8.8-small expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: x86_64-enterprise-rhel8 - compile_flags: >- - --ssl MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: --excludeWithAnyTags=requires_external_data_source - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" - CFLAGS="-fno-gnu-unique" multiversion_platform: rhel80 multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise @@ -116,7 +104,6 @@ buildvariants: packager_arch: x86_64 packager_distro: rhel88 repo_edition: enterprise - scons_cache_scope: shared jstestfuzz_num_generated_files: 40 jstestfuzz_concurrent_num_files: 10 target_resmoke_time: 10 @@ -157,13 +144,11 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: aarch64-rhel8 - compile_flags: >- - --ssl MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source,requires_increased_memlock_limits --enableEnterpriseTests=off @@ -172,7 +157,6 @@ buildvariants: packager_arch: aarch64 packager_distro: rhel88 repo_edition: org - scons_cache_scope: shared large_distro_name: rhel8.8-arm64-large compile_variant: rhel-8-arm64 tasks: @@ -199,29 +183,22 @@ buildvariants: - rhel8.8-arm64-small expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: aarch64-enterprise-rhel8 - compile_flags: >- - --ssl MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: --excludeWithAnyTags=requires_external_data_source,resource_intensive - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic - -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" has_packages: true packager_script: packager_enterprise.py packager_arch: aarch64 packager_distro: rhel88 repo_edition: enterprise - scons_cache_scope: shared compile_variant: enterprise-rhel-8-arm64 core_analyzer_distro_name: rhel8.8-arm64-large tasks: @@ -254,13 +231,11 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-rhel93 - compile_flags: >- - --ssl MONGO_DISTMOD=rhel93 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel93 + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source --enableEnterpriseTests=off @@ -271,7 +246,6 @@ buildvariants: packager_arch: x86_64 packager_distro: rhel90 repo_edition: org - scons_cache_scope: shared large_distro_name: rhel93-large compile_variant: rhel93 tasks: @@ -298,16 +272,15 @@ buildvariants: run_on: - rhel93-small expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug + additional_package_targets: archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: x86_64-enterprise-rhel93 - compile_flags: --ssl MONGO_DISTMOD=rhel93 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: --define=MONGO_DISTMOD=rhel93 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: --excludeWithAnyTags=requires_external_data_source - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" multiversion_platform: rhel93 multiversion_edition: enterprise has_packages: true @@ -315,7 +288,6 @@ buildvariants: packager_arch: x86_64 packager_distro: rhel90 repo_edition: enterprise - scons_cache_scope: shared compile_variant: enterprise-rhel-93-64-bit core_analyzer_distro_name: rhel93-large large_distro_name: rhel93-large @@ -350,13 +322,11 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: aarch64-rhel93 - compile_flags: >- - --ssl MONGO_DISTMOD=rhel93 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel93 + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source --enableEnterpriseTests=off @@ -365,7 +335,6 @@ buildvariants: packager_arch: aarch64 packager_distro: rhel90 repo_edition: org - scons_cache_scope: shared large_distro_name: rhel93-arm64-large compile_variant: rhel93-arm64 tasks: @@ -395,29 +364,22 @@ buildvariants: - rhel93-arm64-small expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: aarch64-enterprise-rhel93 - compile_flags: >- - --ssl MONGO_DISTMOD=rhel93 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel93 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: --excludeWithAnyTags=requires_external_data_source - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic - -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" has_packages: true packager_script: packager_enterprise.py packager_arch: aarch64 packager_distro: rhel90 repo_edition: enterprise - scons_cache_scope: shared large_distro_name: rhel93-arm64-large compile_variant: enterprise-rhel-93-arm64 tasks: diff --git a/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml b/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml index c56906c3bb3..1e850de4061 100644 --- a/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml +++ b/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml @@ -27,8 +27,6 @@ variables: # - etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &generic_linux_compile_expansions # The most commonly used compile expansions. - scons_cache_scope: shared - scons_cache_mode: all has_packages: false # THIS HAS COPIES IN: @@ -54,7 +52,6 @@ variables: repo_edition: enterprise large_distro_name: rhel8.8-medium core_analyzer_distro_name: rhel8.8-xlarge - num_scons_link_jobs_available: 0.99 compile_variant: *linux_x86_dynamic_compile_variant_name # THIS HAS COPIES IN: @@ -65,8 +62,6 @@ variables: # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &enterprise-rhel-8-64-bit-dynamic-expansions <<: *linux_x86_generic_expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false jstestfuzz_num_generated_files: 40 jstestfuzz_concurrent_num_files: 10 @@ -82,16 +77,12 @@ variables: # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &linux_debug_community_compile_expansions <<: *generic_linux_compile_expansions - compile_flags: >- - --dbg=on + bazel_compile_flags: >- + --dbg=True --opt=on - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --enable-http-client=on - --link-model=dynamic - ENABLE_OTEL_BUILD=1 - --modules= - num_scons_link_jobs_available: 0.99 + --linkstatic=False + --build_otel=True + --build_enterprise=False - &linux_x86_dynamic_debug_compile_variant_dependency depends_on: @@ -124,7 +115,7 @@ variables: # variant: linux-debug-aubsan-compile-required # If you add anything to san_options, make sure the appropriate changes are - # also made to SConstruct. + # also made to in bazel. # THIS HAS COPIES IN: # - etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml # - etc/evergreen_yml_components/variants/sanitizer/test_dev.yml @@ -146,7 +137,6 @@ variables: resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under {A,UB}SAN build. hang_analyzer_dump_core: false max_sub_suites: 3 - num_scons_link_jobs_available: 0.99 large_distro_name: rhel8.8-large xlarge_distro_name: rhel8.8-xxlarge multiversion_platform: rhel80 @@ -175,13 +165,13 @@ variables: # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &enterprise-rhel8-debug-tsan-expansions-template additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug lang_environment: LANG=C toolchain_version: stable compile_variant: *enterprise-rhel8-debug-tsan-compile # If you add anything to san_options, make sure the appropriate - # changes are also made to SConstruct. + # changes are also made in bazel. # # TODO SERVER-49121: die_after_fork=0 is a temporary setting to # allow tests to continue while we figure out why we're running @@ -194,20 +184,12 @@ variables: san_options: TSAN_OPTIONS="abort_on_error=1:disable_coredump=0:handle_abort=1:halt_on_error=1:report_thread_leaks=0:die_after_fork=0:history_size=4:suppressions=etc/tsan.suppressions:external_symbolizer_path=/opt/mongodbtoolchain/v5/bin/llvm-symbolizer" build_mongot: true download_mongot_release: true - compile_flags: >- - --variables-files=etc/scons/mongodbtoolchain_${toolchain_version}_clang.vars - --dbg=on + bazel_compile_flags: >- + --config=dbg_tsan --opt=on - --allocator=system - --sanitize=thread - --ssl - --use-libunwind=off - --link-model=dynamic - -j$(grep -c ^processor /proc/cpuinfo) + --fission=no # Avoid starting too many mongod's under TSAN build. resmoke_jobs_factor: 0.3 - scons_cache_scope: shared - separate_debug: off large_distro_name: &enterprise-rhel8-debug-tsan-large-distro-name rhel8.8-xlarge xlarge_distro_name: rhel8.8-xxlarge core_analyzer_distro_name: rhel8.8-xxlarge @@ -251,17 +233,11 @@ buildvariants: tags: ["required", "forbid_tasks_tagged_with_experimental"] expansions: <<: *generic_linux_compile_expansions - separate_debug: off san_options: *aubsan_options - compile_flags: >- - --variables-files=etc/scons/mongodbtoolchain_stable_clang.vars - --dbg=on + bazel_compile_flags: >- + --config=dbg_aubsan --opt=on - --allocator=system - --sanitize=undefined,address - --ssl - -j$(grep -c ^processor /proc/cpuinfo) - --link-model=dynamic + --fission=no build_mongot: true download_mongot_release: true evergreen_remote_exec: on @@ -391,25 +367,22 @@ buildvariants: expansions: lang_environment: LANG=C san_options: *aubsan_options - compile_flags: >- - --variables-files=etc/scons/mongodbtoolchain_stable_clang.vars + bazel_compile_flags: >- --opt=on --allocator=system - --sanitize=address - --ssl - -j$(grep -c ^processor /proc/cpuinfo) - --link-model=dynamic - --modules= + --asan=True + --compiler_type=clang + --linkstatic=False + --build_enterprise=False + --separate_debug=False multiversion_platform: rhel80 multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under ASAN build. hang_analyzer_dump_core: false - scons_cache_scope: shared test_flags: >- --excludeWithAnyTags=requires_fast_memory,corrupts_data --enableEnterpriseTests=off - separate_debug: off compile_variant: rhel8-asan tasks: - name: compile_test_serial_TG @@ -427,30 +400,23 @@ buildvariants: stepback: false expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug lang_environment: LANG=C toolchain_version: stable # If you add anything to san_options, make sure the appropriate changes are - # also made to SConstruct. + # also made in bazel san_options: *aubsan_options - compile_flags: >- - LINKFLAGS=-nostdlib++ - LIBS=stdc++ - --variables-files=etc/scons/mongodbtoolchain_${toolchain_version}_clang.vars - --dbg=on + bazel_compile_flags: >- + --config=dbg_aubsan --opt=on - --allocator=system - --sanitize=undefined,address,fuzzer - --ssl - -j$(grep -c ^processor /proc/cpuinfo) + --fission=no + --fsan=True test_flags: >- --mongodSetParameters="{internalQueryEnableAggressiveSpillsInGroup: true}" --excludeWithAnyTags=corrupts_data resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under {A,UB}SAN build. hang_analyzer_dump_core: false - scons_cache_scope: shared - separate_debug: off compile_variant: *rhel8-debug-aubsan-lite_fuzzer display_tasks: - name: libfuzzertests! @@ -596,13 +562,10 @@ buildvariants: # These modes correspond to a GC policy of generationalGC (mode 7) every 75 allocations, and a # consistency check of the heap after every GC cycle (mode 15). mongo_mozjs_options: "7;15,75" - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --link-model=dynamic - --spider-monkey-dbg=on + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 + --linkstatic=False + --spider_monkey_dbg=True compile_variant: enterprise-rhel-8-64-bit-dynamic-spider-monkey-dbg exec_timeout_secs: 32400 # 9 hour timeout timeout_secs: 18000 # 5 hour idle timeout diff --git a/etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml index fb7a07f8e9d..ca56dbbc853 100644 --- a/etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml @@ -12,8 +12,6 @@ variables: # - etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &generic_linux_compile_expansions # The most commonly used compile expansions. - scons_cache_scope: shared - scons_cache_mode: all has_packages: false # THIS HAS COPIES IN: @@ -57,7 +55,6 @@ variables: repo_edition: enterprise large_distro_name: rhel8.8-medium core_analyzer_distro_name: rhel8.8-xlarge - num_scons_link_jobs_available: 0.99 compile_variant: *linux_x86_dynamic_compile_variant_name # THIS HAS COPIES IN: @@ -68,8 +65,6 @@ variables: # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &enterprise-rhel-8-64-bit-dynamic-expansions <<: *linux_x86_generic_expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false jstestfuzz_num_generated_files: 40 jstestfuzz_concurrent_num_files: 10 @@ -80,7 +75,7 @@ variables: large_distro_name: rhel8.8-medium # If you add anything to san_options, make sure the appropriate changes are - # also made to SConstruct. + # also made in bazel. # THIS HAS COPIES IN: # - etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml # - etc/evergreen_yml_components/variants/sanitizer/test_dev.yml @@ -111,7 +106,6 @@ variables: resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under {A,UB}SAN build. hang_analyzer_dump_core: false max_sub_suites: 3 - num_scons_link_jobs_available: 0.99 large_distro_name: rhel8.8-large xlarge_distro_name: rhel8.8-xxlarge multiversion_platform: rhel80 @@ -140,13 +134,13 @@ variables: # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &enterprise-rhel8-debug-tsan-expansions-template additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug lang_environment: LANG=C toolchain_version: stable compile_variant: *enterprise-rhel8-debug-tsan-compile # If you add anything to san_options, make sure the appropriate - # changes are also made to SConstruct. + # changes are also made in bazel. # # TODO SERVER-49121: die_after_fork=0 is a temporary setting to # allow tests to continue while we figure out why we're running @@ -159,20 +153,12 @@ variables: san_options: TSAN_OPTIONS="abort_on_error=1:disable_coredump=0:handle_abort=1:halt_on_error=1:report_thread_leaks=0:die_after_fork=0:history_size=4:suppressions=etc/tsan.suppressions:external_symbolizer_path=/opt/mongodbtoolchain/v5/bin/llvm-symbolizer" build_mongot: true download_mongot_release: true - compile_flags: >- - --variables-files=etc/scons/mongodbtoolchain_${toolchain_version}_clang.vars - --dbg=on + bazel_compile_flags: >- + --config=dbg_tsan --opt=on - --allocator=system - --sanitize=thread - --ssl - --use-libunwind=off - --link-model=dynamic - -j$(grep -c ^processor /proc/cpuinfo) + --fission=no # Avoid starting too many mongod's under TSAN build. resmoke_jobs_factor: 0.3 - scons_cache_scope: shared - separate_debug: off large_distro_name: &enterprise-rhel8-debug-tsan-large-distro-name rhel8.8-xlarge xlarge_distro_name: rhel8.8-xxlarge core_analyzer_distro_name: rhel8.8-xxlarge @@ -198,14 +184,11 @@ buildvariants: test_flags: >- --mongodSetParameters="{internalQueryEnableAggressiveSpillsInGroup: true}" --excludeWithAnyTags=incompatible_slow_machine - compile_flags: >- - --ssl - MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --link-model=dynamic - --use-glibcxx-debug - --dbg=on + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 + --linkstatic=False + --use_glibcxx_debug=True + --dbg=True --allocator=system exec_timeout_secs: 32400 # 9 hour timeout timeout_secs: 18000 # 5 hour idle timeout @@ -271,17 +254,13 @@ buildvariants: expansions: exe: ".exe" content_type: application/zip - compile_flags: >- - --dbg=on + bazel_compile_flags: >- + --dbg=True --opt=on - --win-version-min=win10 - -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.8") - MONGO_DISTMOD=windows - --modules= - num_scons_link_jobs_available: 0.2 + --define=MONGO_DISTMOD=windows + --build_enterprise=False python: "/cygdrive/c/python/python310/python.exe" ext: zip - scons_cache_scope: shared multiversion_platform: windows multiversion_edition: enterprise large_distro_name: windows-2022-large @@ -318,22 +297,17 @@ buildvariants: expansions: compile_variant: *enterprise-windows-debug-unoptimized additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug exe: ".exe" content_type: application/zip - compile_flags: >- - --dbg=on + bazel_compile_flags: >- + --dbg=True --opt=off - --ssl - MONGO_DISTMOD=windows - -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") - --win-version-min=win10 - num_scons_link_jobs_available: 0.25 + --define=MONGO_DISTMOD=windows python: "/cygdrive/c/python/python310/python.exe" ext: zip resmoke_jobs_max: 1 - scons_cache_scope: shared test_flags: >- --excludeWithAnyTags=incompatible_with_windows_tls --mongodSetParameters="{internalQueryEnableAggressiveSpillsInGroup: true}" @@ -364,16 +338,11 @@ buildvariants: --enableEnterpriseTests=off --mongodSetParameters="{internalQueryEnableAggressiveSpillsInGroup: true}" resmoke_jobs_max: 6 - compile_env: DEVELOPER_DIR=/Applications/Xcode15.app - compile_flags: >- - --ssl - --dbg=on + bazel_compile_flags: >- + --dbg=True --opt=on - -j$(sysctl -n hw.logicalcpu) - --libc++ - --variables-files=etc/scons/xcode_macosx_arm.vars - --modules= - num_scons_link_jobs_available: 0.99 + --build_enterprise=False + --developer_dir=/Applications/Xcode15.app tasks: - name: compile_test_serial_TG - name: compile_test_parallel_unittest_stream_TG @@ -397,16 +366,11 @@ buildvariants: test_flags: >- --mongodSetParameters="{internalQueryEnableAggressiveSpillsInGroup: true}" --excludeWithAnyTags=incompatible_with_macos,requires_gcm - compile_env: DEVELOPER_DIR=/Applications/Xcode15.app - compile_flags: >- - --ssl - --dbg=on + bazel_compile_flags: >- + --dbg=True --opt=on - -j$(sysctl -n hw.logicalcpu) - --libc++ - --variables-files=etc/scons/xcode_macosx_arm.vars + --developer_dir=/Applications/Xcode15.app resmoke_jobs_max: 6 - num_scons_link_jobs_available: 0.99 tasks: - name: compile_test_serial_TG - name: compile_test_parallel_unittest_stream_TG @@ -433,18 +397,17 @@ buildvariants: expansions: toolchain_version: stable additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug lang_environment: LANG=C san_options: *ubsan_options - compile_flags: >- - --variables-files=etc/scons/mongodbtoolchain_${toolchain_version}_clang.vars - --dbg=on + bazel_compile_flags: >- + --dbg=True --opt=on - --sanitize=undefined - --ssl + --ubsan=True + --compiler_type=clang --allocator=tcmalloc-gperf - -j$(grep -c ^processor /proc/cpuinfo) + --separate_debug=False compile_variant: *rhel8-debug-ubsan-all-feature-flags # To force disable feature flags even on the all feature flags variant, please use this file: # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml @@ -456,8 +419,6 @@ buildvariants: multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under UBSAN build. - scons_cache_scope: shared - separate_debug: off large_distro_name: rhel8.8-large tasks: - name: compile_and_archive_dist_test_TG @@ -530,17 +491,11 @@ buildvariants: - rhel8.8-large expansions: <<: *generic_linux_compile_expansions - separate_debug: off san_options: *aubsan_options - compile_flags: >- - --variables-files=etc/scons/mongodbtoolchain_stable_clang.vars - --dbg=on + bazel_compile_flags: >- + --config=dbg_aubsan --opt=on - --allocator=system - --sanitize=undefined,address - --ssl - -j$(grep -c ^processor /proc/cpuinfo) - --link-model=dynamic + --fission=no test_flags: >- --benchmarkRepetitions=1 --excludeWithAnyTags=incompatible_aubsan @@ -601,18 +556,13 @@ buildvariants: tags: ["bazel_check"] cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. expansions: - compile_flags: >- - MONGO_DISTMOD=rhel88 - -j$(grep -c ^processor /proc/cpuinfo) - --dbg=on + bazel_compile_flags: >- + --define=MONGO_DISTMOD=rhel88 + --dbg=True --opt=off - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --detect-odr-violations + --detect_odr_violations=True --linker=gold - --link-model=dynamic - scons_cache_mode: all - scons_cache_scope: shared - num_scons_link_jobs_available: 0.99 + --linkstatic=False compile_variant: *enterprise-rhel8-unoptimized-64-bit tasks: - name: compile_test_parallel_core_stream_TG @@ -636,21 +586,13 @@ buildvariants: - rhel8.8-large expansions: <<: *aubsan-lite-required-expansions - scons_cache_scope: shared - scons_cache_mode: all has_packages: false - separate_debug: off san_options: *grpc_aubsan_options - compile_flags: >- - --variables-files=etc/scons/mongodbtoolchain_stable_clang.vars - --dbg=on + bazel_compile_flags: >- + --config=dbg_aubsan --opt=on - --allocator=system - --sanitize=undefined,address - --ssl - --ocsp-stapling=off - -j$(grep -c ^processor /proc/cpuinfo) - --link-model=dynamic + --fission=no + --use_ocsp_stapling=False compile_variant: *linux-debug-aubsan-compile-grpc large_distro_name: rhel80-xlarge test_flags: >- diff --git a/etc/evergreen_yml_components/variants/suse/test_release.yml b/etc/evergreen_yml_components/variants/suse/test_release.yml index fffdc778125..ea401b5b59f 100644 --- a/etc/evergreen_yml_components/variants/suse/test_release.yml +++ b/etc/evergreen_yml_components/variants/suse/test_release.yml @@ -13,30 +13,23 @@ buildvariants: - suse15sp5-small expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: x86_64-enterprise-suse15 - compile_flags: >- - --ssl - MONGO_DISTMOD=suse15 - BAZEL_FLAGS="--build_otel=False" - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=suse15 + --build_otel=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: --excludeWithAnyTags=requires_external_data_source,requires_increased_memlock_limits,requires_ldap_pool,resource_intensive - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" has_packages: true packager_script: packager_enterprise.py packager_arch: x86_64 packager_distro: suse15 repo_edition: enterprise - scons_cache_scope: shared compile_variant: enterprise-suse15-64 core_analyzer_distro_name: suse15sp5-large core_analyzer_gdb_index_cache: off @@ -67,15 +60,12 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-suse15 - compile_flags: >- - --ssl - MONGO_DISTMOD=suse15 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - BAZEL_FLAGS="--build_otel=False" - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=suse15 + --build_otel=False + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source,requires_increased_memlock_limits --enableEnterpriseTests=off @@ -86,7 +76,6 @@ buildvariants: packager_arch: x86_64 packager_distro: suse15 repo_edition: org - scons_cache_scope: shared large_distro_name: suse15sp5-large compile_variant: suse15 core_analyzer_gdb_index_cache: off diff --git a/etc/evergreen_yml_components/variants/ubuntu/test_dev_master_and_lts_branches_only.yml b/etc/evergreen_yml_components/variants/ubuntu/test_dev_master_and_lts_branches_only.yml index 773621798a4..cb1f693b633 100644 --- a/etc/evergreen_yml_components/variants/ubuntu/test_dev_master_and_lts_branches_only.yml +++ b/etc/evergreen_yml_components/variants/ubuntu/test_dev_master_and_lts_branches_only.yml @@ -14,7 +14,6 @@ variables: multiversion_edition: enterprise repo_edition: enterprise large_distro_name: ubuntu2204-large - scons_cache_scope: shared tasks: - name: compile_and_archive_dist_test_TG - name: .antithesis @@ -29,14 +28,11 @@ buildvariants: expansions: <<: *enterprise-ubuntu2204-64-libvoidstar-expansions-template compile_variant: *enterprise-ubuntu2204-64-libvoidstar - compile_flags: >- - --ssl - MONGO_DISTMOD=ubuntu2204 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_clang.vars - BAZEL_FLAGS="--copt=-fsanitize-coverage=trace-pc-guard --linkopt=-lvoidstar" - CCFLAGS="-fsanitize-coverage=trace-pc-guard" - LIBS="voidstar" + bazel_compile_flags: >- + --define=MONGO_DISTMOD=ubuntu2204 + --compiler_type=clang + --copt=-fsanitize-coverage=trace-pc-guard + --linkopt=-lvoidstar - <<: *enterprise-ubuntu2204-64-libvoidstar-template name: &enterprise-ubuntu2204-64-aubsan-libvoidstar enterprise-ubuntu2204-64-aubsan-libvoidstar @@ -47,21 +43,14 @@ buildvariants: <<: *enterprise-ubuntu2204-64-libvoidstar-expansions-template antithesis_build_type: aubsan compile_variant: *enterprise-ubuntu2204-64-aubsan-libvoidstar - separate_debug: off - compile_flags: >- - --allocator=system - --sanitize=address,undefined - --ssl + bazel_compile_flags: >- + --config=dbg_aubsan --opt=on - --dbg=on - MONGO_DISTMOD=ubuntu2204 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_clang.vars - --shared-libsan - --link-model=dynamic - BAZEL_FLAGS="--copt=-fsanitize-coverage=trace-pc-guard --linkopt=-lvoidstar" - CCFLAGS="-fsanitize-coverage=trace-pc-guard" - LIBS="voidstar" + --fission=no + --define=MONGO_DISTMOD=ubuntu2204 + --copt=-shared-libsan + --copt=-fsanitize-coverage=trace-pc-guard + --linkopt=-lvoidstar # TODO SERVER-100334 Enable detect_stack_use_after_return san_options: >- UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/usr/lib/llvm-12/bin/llvm-symbolizer" @@ -81,21 +70,13 @@ buildvariants: <<: *enterprise-ubuntu2204-64-libvoidstar-expansions-template antithesis_build_type: tsan compile_variant: *enterprise-ubuntu2204-64-tsan-libvoidstar - separate_debug: off - compile_flags: >- - --allocator=system - --sanitize=thread - --use-libunwind=off + bazel_compile_flags: >- + --config=dbg_tsan --opt=on - --dbg=on - --ssl - MONGO_DISTMOD=ubuntu2204 - --link-model=dynamic - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_clang.vars - BAZEL_FLAGS="--copt=-fsanitize-coverage=trace-pc-guard --linkopt=-lvoidstar" - CCFLAGS="-fsanitize-coverage=trace-pc-guard" - LIBS="voidstar" + --fission=no + --define=MONGO_DISTMOD=ubuntu2204 + --copt=-fsanitize-coverage=trace-pc-guard + --linkopt=-lvoidstar san_options: TSAN_OPTIONS="abort_on_error=1:disable_coredump=0:handle_abort=1:halt_on_error=1:report_thread_leaks=0:die_after_fork=0:history_size=4" test_flags: >- --excludeWithAnyTags=tsan_incompatible diff --git a/etc/evergreen_yml_components/variants/ubuntu/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/ubuntu/test_dev_master_branch_only.yml index e3755ff753c..c8b3d086608 100644 --- a/etc/evergreen_yml_components/variants/ubuntu/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/ubuntu/test_dev_master_branch_only.yml @@ -5,23 +5,6 @@ # and LTS release (v7.0, v6.0 etc.) branch projects buildvariants: - - name: &ubuntu2204-arm64-bazel-compile ubuntu2204-arm64-bazel-compile - display_name: "* Ubuntu 22.04 arm64 Enterprise Bazel Compile" - tags: ["suggested", "bazel_check"] - cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter - run_on: - - ubuntu2204-arm64-large - stepback: false - expansions: - scons_cache_scope: shared - compile_variant: *ubuntu2204-arm64-bazel-compile - tasks: - - name: compile_bazel_TG - - name: run_bazel_TG - - name: bazel_workstation_TG - distros: - - ubuntu2204-arm64-large - - name: &ubuntu2204-x86-64-bazel-compile ubuntu2204-x86-64-bazel-compile display_name: "Ubuntu 22.04 x86_64 Enterprise Bazel Compile" tags: ["bazel_check"] diff --git a/etc/evergreen_yml_components/variants/ubuntu/test_release.yml b/etc/evergreen_yml_components/variants/ubuntu/test_release.yml index fc4f38a870d..ecd9a9c68d0 100644 --- a/etc/evergreen_yml_components/variants/ubuntu/test_release.yml +++ b/etc/evergreen_yml_components/variants/ubuntu/test_release.yml @@ -15,16 +15,12 @@ buildvariants: # TODO SERVER-86538 see if we can use --allocator=tcmalloc-google on # new distributions and/or fix the current libfaketime issues expansions: - compile_flags: >- - --ssl - MONGO_DISTMOD=ubuntu1804 + bazel_compile_flags: >- + --define=MONGO_DISTMOD=ubuntu1804 --allocator=tcmalloc-gperf - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --disable-streams + --disable_streams=True compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" - scons_cache_scope: shared + --linkopt=-s large_distro_name: ubuntu1804-large compile_variant: enterprise-ubuntu1804-64 tasks: @@ -47,14 +43,11 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-ubuntu2204 - compile_flags: >- - --ssl - MONGO_DISTMOD=ubuntu2204 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=ubuntu2204 + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source,requires_increased_memlock_limits --enableEnterpriseTests=off @@ -65,7 +58,6 @@ buildvariants: packager_arch: x86_64 packager_distro: ubuntu2204 repo_edition: org - scons_cache_scope: shared large_distro_name: ubuntu2204-large compile_variant: ubuntu2204 tasks: @@ -106,14 +98,11 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-ubuntu2404 - compile_flags: >- - --ssl - MONGO_DISTMOD=ubuntu2404 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=ubuntu2404 + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source,requires_increased_memlock_limits,requires_updated_oscrypto --enableEnterpriseTests=off @@ -124,7 +113,6 @@ buildvariants: packager_arch: x86_64 packager_distro: ubuntu2404 repo_edition: org - scons_cache_scope: shared large_distro_name: ubuntu2404-large compile_variant: ubuntu2404 tasks: @@ -164,14 +152,11 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-ubuntu2004 - compile_flags: >- - --ssl - MONGO_DISTMOD=ubuntu2004 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=ubuntu2004 + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source --enableEnterpriseTests=off @@ -182,7 +167,6 @@ buildvariants: packager_arch: x86_64 packager_distro: ubuntu2004 repo_edition: org - scons_cache_scope: shared large_distro_name: ubuntu2004-large compile_variant: ubuntu2004 tasks: @@ -211,22 +195,17 @@ buildvariants: stepback: false expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: x86_64-enterprise-ubuntu2004 - compile_flags: >- - --ssl MONGO_DISTMOD=ubuntu2004 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=ubuntu2004 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: --excludeWithAnyTags=requires_external_data_source - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" multiversion_platform: ubuntu2004 multiversion_edition: enterprise has_packages: true @@ -234,7 +213,6 @@ buildvariants: packager_arch: x86_64 packager_distro: ubuntu2004 repo_edition: enterprise - scons_cache_scope: shared large_distro_name: ubuntu2004-large compile_variant: enterprise-ubuntu2004-64 tasks: @@ -276,23 +254,17 @@ buildvariants: stepback: false expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: x86_64-enterprise-ubuntu2204 - compile_flags: >- - --ssl - MONGO_DISTMOD=ubuntu2204 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=ubuntu2204 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: --excludeWithAnyTags=requires_external_data_source - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" multiversion_platform: ubuntu2204 multiversion_edition: enterprise has_packages: true @@ -300,7 +272,6 @@ buildvariants: packager_arch: x86_64 packager_distro: ubuntu2204 repo_edition: enterprise - scons_cache_scope: shared large_distro_name: ubuntu2204-large compile_variant: enterprise-ubuntu2204-64 tasks: @@ -330,12 +301,8 @@ buildvariants: - ubuntu2204-small stepback: false expansions: - compile_flags: >- - --ssl - MONGO_DISTMOD=ubuntu2204 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - scons_cache_scope: shared + bazel_compile_flags: >- + --define=MONGO_DISTMOD=ubuntu2204 large_distro_name: ubuntu2204-large compile_variant: enterprise-ubuntu2204-64 tasks: @@ -354,22 +321,17 @@ buildvariants: - ubuntu2004-arm64 expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: aarch64-enterprise-ubuntu2004 - compile_flags: >- - --ssl MONGO_DISTMOD=ubuntu2004 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=ubuntu2004 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: --excludeWithAnyTags=requires_external_data_source - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" resmoke_jobs_max: 4 # Avoid starting too many mongod's on ARM test servers has_packages: true packager_script: packager_enterprise.py @@ -379,7 +341,6 @@ buildvariants: multiversion_platform: ubuntu2004 multiversion_architecture: aarch64 multiversion_edition: enterprise - scons_cache_scope: shared compile_variant: enterprise-ubuntu2004-arm64 large_distro_name: ubuntu2004-arm64-large tasks: @@ -409,14 +370,11 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: aarch64-ubuntu2004 - compile_flags: >- - --ssl - MONGO_DISTMOD=ubuntu2004 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=ubuntu2004 + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source --enableEnterpriseTests=off @@ -429,7 +387,6 @@ buildvariants: multiversion_platform: ubuntu2004 multiversion_architecture: aarch64 multiversion_edition: targeted - scons_cache_scope: shared compile_variant: ubuntu2004-arm64 large_distro_name: ubuntu2004-arm64-large tasks: @@ -455,16 +412,15 @@ buildvariants: run_on: - ubuntu2204-arm64-small expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug + additional_package_targets: archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: aarch64-enterprise-ubuntu2204 - compile_flags: --ssl MONGO_DISTMOD=ubuntu2204 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: --define=MONGO_DISTMOD=ubuntu2204 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: --excludeWithAnyTags=requires_external_data_source - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" resmoke_jobs_max: 4 # Avoid starting too many mongod's on ARM test servers has_packages: true packager_script: packager_enterprise.py @@ -474,7 +430,6 @@ buildvariants: multiversion_platform: ubuntu2204 multiversion_architecture: aarch64 multiversion_edition: enterprise - scons_cache_scope: shared compile_variant: enterprise-ubuntu2204-arm64 large_distro_name: ubuntu2204-arm64-large tasks: @@ -507,13 +462,11 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: aarch64-ubuntu2204 - compile_flags: >- - --ssl MONGO_DISTMOD=ubuntu2204 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=ubuntu2204 + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source --enableEnterpriseTests=off @@ -526,7 +479,6 @@ buildvariants: multiversion_platform: ubuntu2204 multiversion_architecture: aarch64 multiversion_edition: targeted - scons_cache_scope: shared compile_variant: ubuntu2204-arm64 large_distro_name: ubuntu2204-arm64-large tasks: @@ -554,23 +506,17 @@ buildvariants: stepback: false expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: x86_64-enterprise-ubuntu2404 - compile_flags: >- - --ssl - MONGO_DISTMOD=ubuntu2404 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: >- + --define=MONGO_DISTMOD=ubuntu2404 compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: --excludeWithAnyTags=requires_external_data_source,requires_updated_oscrypto - crypt_task_compile_flags: >- - SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" - CCFLAGS="-fno-gnu-unique" multiversion_platform: ubuntu2404 multiversion_edition: enterprise has_packages: true @@ -578,7 +524,6 @@ buildvariants: packager_arch: x86_64 packager_distro: ubuntu2404 repo_edition: enterprise - scons_cache_scope: shared large_distro_name: ubuntu2404-large compile_variant: enterprise-ubuntu2404 tasks: @@ -611,13 +556,11 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: aarch64-ubuntu2404 - compile_flags: >- - --ssl MONGO_DISTMOD=ubuntu2404 - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --modules= + bazel_compile_flags: >- + --define=MONGO_DISTMOD=ubuntu2404 + --build_enterprise=False compile_all_but_not_unittests_flags: >- - BAZEL_FLAGS="--linkopt=-s" + --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source,requires_updated_oscrypto --enableEnterpriseTests=off @@ -630,7 +573,6 @@ buildvariants: multiversion_platform: ubuntu2404 multiversion_architecture: aarch64 multiversion_edition: targeted - scons_cache_scope: shared compile_variant: ubuntu2404-arm64 large_distro_name: ubuntu2404-arm64-large tasks: @@ -656,14 +598,13 @@ buildvariants: run_on: - ubuntu2404-arm64-small expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug + additional_package_targets: archive-mongocryptd-stripped archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: aarch64-enterprise-ubuntu2404 - compile_flags: --ssl MONGO_DISTMOD=ubuntu2404 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + bazel_compile_flags: --define=MONGO_DISTMOD=ubuntu2404 test_flags: --excludeWithAnyTags=requires_external_data_source,requires_updated_oscrypto - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" resmoke_jobs_max: 4 # Avoid starting too many mongod's on ARM test servers has_packages: true packager_script: packager_enterprise.py @@ -673,7 +614,6 @@ buildvariants: multiversion_platform: ubuntu2404 multiversion_architecture: aarch64 multiversion_edition: enterprise - scons_cache_scope: shared compile_variant: enterprise-ubuntu2404-arm64 large_distro_name: ubuntu2404-arm64-large tasks: diff --git a/etc/evergreen_yml_components/variants/windows/test_dev.yml b/etc/evergreen_yml_components/variants/windows/test_dev.yml index 94073b477f0..ee2303cb394 100644 --- a/etc/evergreen_yml_components/variants/windows/test_dev.yml +++ b/etc/evergreen_yml_components/variants/windows/test_dev.yml @@ -56,18 +56,13 @@ buildvariants: exe: ".exe" ext: zip additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug - msi + //src/mongo/installer/msi:mongodb-win32-x86_64-windows content_type: application/zip - compile_flags: >- - --ssl - MONGO_DISTMOD=windows - -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") - --win-version-min=win10 - num_scons_link_jobs_available: 0.1 + bazel_compile_flags: >- + --define=MONGO_DISTMOD=windows python: "/cygdrive/c/python/python310/python.exe" - scons_cache_scope: shared compile_variant: *windows-compile-required tasks: - name: compile_test_serial_no_unittests_TG diff --git a/etc/evergreen_yml_components/variants/windows/test_dev_master_and_lts_branches_only.yml b/etc/evergreen_yml_components/variants/windows/test_dev_master_and_lts_branches_only.yml index 4ed65fa83f2..6e07158eba5 100644 --- a/etc/evergreen_yml_components/variants/windows/test_dev_master_and_lts_branches_only.yml +++ b/etc/evergreen_yml_components/variants/windows/test_dev_master_and_lts_branches_only.yml @@ -13,23 +13,18 @@ buildvariants: - windows-2022-small expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug - msi + //src/mongo/installer/msi:mongodb-win32-x86_64-windows exe: ".exe" content_type: application/zip - compile_flags: >- - --ssl - MONGO_DISTMOD=windows - -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") - --win-version-min=win10 - num_scons_link_jobs_available: 0.1 + bazel_compile_flags: >- + --define=MONGO_DISTMOD=windows python: "/cygdrive/c/python/python310/python.exe" test_flags: >- --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling,incompatible_with_windows_tls ext: zip - scons_cache_scope: shared multiversion_platform: windows multiversion_edition: enterprise large_distro_name: windows-2022-large diff --git a/etc/evergreen_yml_components/variants/windows/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/windows/test_dev_master_branch_only.yml index 53cb660deef..bd0522d9027 100644 --- a/etc/evergreen_yml_components/variants/windows/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/windows/test_dev_master_branch_only.yml @@ -59,11 +59,8 @@ buildvariants: exe: ".exe" ext: zip content_type: application/zip - compile_flags: >- - --ssl - MONGO_DISTMOD=windows - -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.8") - --win-version-min=win10 + bazel_compile_flags: >- + --define=MONGO_DISTMOD=windows python: "/cygdrive/c/python/python310/python.exe" compile_variant: *windows-bazel-compile tasks: @@ -78,20 +75,15 @@ buildvariants: - windows-2022-small expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug - msi + //src/mongo/installer/msi:mongodb-win32-x86_64-windows exe: ".exe" content_type: application/zip - compile_flags: >- - --ssl - MONGO_DISTMOD=windows - -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) - --win-version-min=win10 - num_scons_link_jobs_available: 0.5 + bazel_compile_flags: >- + --define=MONGO_DISTMOD=windows python: "/cygdrive/c/python/python310/python.exe" ext: zip - scons_cache_scope: shared multiversion_platform: windows multiversion_edition: enterprise jstestfuzz_num_generated_files: 35 @@ -100,7 +92,7 @@ buildvariants: external_auth_jobs_max: 1 compile_variant: *windows-compile-query-patch-only tasks: - - name: compile_dist_test_TG + - name: archive_dist_test_TG distros: - windows-2022-xxlarge @@ -112,20 +104,15 @@ buildvariants: - windows-2022-xxlarge # To accommodate OOD issues noted in BF-31698 expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug - msi + //src/mongo/installer/msi:mongodb-win32-x86_64-windows exe: ".exe" content_type: application/zip - compile_flags: >- - --ssl - MONGO_DISTMOD=windows - -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) - --win-version-min=win10 - num_scons_link_jobs_available: 0.25 + bazel_compile_flags: >- + --define=MONGO_DISTMOD=windows python: "/cygdrive/c/python/python310/python.exe" ext: zip - scons_cache_scope: shared multiversion_platform: windows multiversion_edition: enterprise jstestfuzz_num_generated_files: 35 diff --git a/etc/evergreen_yml_components/variants/windows/test_release.yml b/etc/evergreen_yml_components/variants/windows/test_release.yml index 5f0193a5571..643241027aa 100644 --- a/etc/evergreen_yml_components/variants/windows/test_release.yml +++ b/etc/evergreen_yml_components/variants/windows/test_release.yml @@ -12,7 +12,7 @@ buildvariants: run_on: - windows-2022-small expansions: - additional_package_targets: msi + additional_package_targets: //src/mongo/installer/msi:mongodb-win32-x86_64-windows exe: ".exe" push_path: windows push_bucket: downloads.mongodb.org @@ -21,16 +21,11 @@ buildvariants: multiversion_platform: windows multiversion_edition: base content_type: application/zip - compile_flags: >- - --ssl - MONGO_DISTMOD=windows - -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") - --win-version-min=win10 - --modules= - num_scons_link_jobs_available: 0.25 + bazel_compile_flags: >- + --define=MONGO_DISTMOD=windows + --build_enterprise=False python: "/cygdrive/c/python/python310/python.exe" ext: zip - scons_cache_scope: shared large_distro_name: windows-2022-large test_flags: >- --excludeWithAnyTags=incompatible_with_windows_tls,requires_external_data_source @@ -59,19 +54,14 @@ buildvariants: expansions: exe: ".exe" additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug - msi + //src/mongo/installer/msi:mongodb-win32-x86_64-windows content_type: application/zip - compile_flags: >- - --ssl - MONGO_DISTMOD=windows - -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") - --win-version-min=win10 - num_scons_link_jobs_available: 0.2 + bazel_compile_flags: >- + --define=MONGO_DISTMOD=windows python: "/cygdrive/c/python/python310/python.exe" ext: zip - scons_cache_scope: shared multiversion_platform: windows multiversion_edition: enterprise jstestfuzz_num_generated_files: 35 diff --git a/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml index a168e68fb49..014e2ed79b4 100644 --- a/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml @@ -27,8 +27,6 @@ variables: # - etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &generic_linux_compile_expansions # The most commonly used compile expansions. - scons_cache_scope: shared - scons_cache_mode: all has_packages: false # THIS HAS COPIES IN @@ -37,15 +35,11 @@ variables: # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &linux_debug_community_compile_expansions <<: *generic_linux_compile_expansions - compile_flags: >- - --dbg=on + bazel_compile_flags: >- + --dbg=True --opt=on - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --enable-http-client=on - --link-model=dynamic - --modules= - num_scons_link_jobs_available: 0.99 + --linkstatic=False + --build_enterprise=False buildvariants: - <<: *generic_linux_compile_params @@ -83,17 +77,11 @@ buildvariants: large_distro_name: rhel8.8-medium max_sub_suites: "5" target_resmoke_time: "15" - compile_flags: >- - --dbg=on + bazel_compile_flags: >- + --dbg=True --opt=on - -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars - --enable-http-client=on - --link-model=dynamic - --modules= - scons_cache_mode: all - scons_cache_scope: shared - num_scons_link_jobs_available: 0.99 + --linkstatic=False + --build_enterprise=False test_flags: >- --excludeWithAnyTags=requires_http_client --enableEnterpriseTests=off @@ -140,20 +128,15 @@ buildvariants: - wtdevelop expansions: additional_package_targets: >- - archive-mongocryptd + archive-mongocryptd-stripped archive-mongocryptd-debug - msi + //src/mongo/installer/msi:mongodb-win32-x86_64-windows exe: ".exe" content_type: application/zip - compile_flags: >- - --ssl - MONGO_DISTMOD=windows - -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) - --win-version-min=win10 - num_scons_link_jobs_available: 0.2 + bazel_compile_flags: >- + --define=MONGO_DISTMOD=windows python: "/cygdrive/c/python/python310/python.exe" ext: zip - scons_cache_scope: shared multiversion_platform: windows multiversion_edition: enterprise jstestfuzz_num_generated_files: 35 diff --git a/etc/expansions.default.yml b/etc/expansions.default.yml index 8b8688d84e2..32b6ae8151b 100644 --- a/etc/expansions.default.yml +++ b/etc/expansions.default.yml @@ -23,7 +23,6 @@ skip_symbolization: "false" install_dir: dist-test/bin is_patch: "false" is_commit_queue: "false" -use_scons_cache: "false" npm_command: jstestfuzz tar: tar cmake_path: /opt/cmake/bin/cmake @@ -32,12 +31,7 @@ jstestfuzz_concurrent_num_files: "10" curator_release: "latest" ext: tgz -# SCons compile expansions. +# Bazel compile expansions. compiling_for_test: "false" install_directory: /data/mongo-install-directory -num_scons_link_jobs_available: "0.25" -num_scons_unit_cc_jobs_available: "0.75" -scons_cache_mode: nolinked -show_scons_timings: "true" -separate_debug: "on" skip_archive: "off" diff --git a/evergreen/bazel_compile.sh b/evergreen/bazel_compile.sh index 8cfe5449074..d8b093d5808 100755 --- a/evergreen/bazel_compile.sh +++ b/evergreen/bazel_compile.sh @@ -23,10 +23,6 @@ activate_venv build_patch_id="${build_patch_id:-${reuse_compile_from}}" if [ -n "${build_patch_id}" ]; then echo "build_patch_id detected, trying to skip task" - if [ "${task_name}" = "compile_dist_test" ] || [ "${task_name}" = "compile_dist_test_half" ]; then - echo "Skipping ${task_name} compile without downloading any files" - exit 0 - fi # On windows we change the extension to zip if [ -z "${ext}" ]; then @@ -38,22 +34,22 @@ if [ -n "${build_patch_id}" ]; then # get the platform of the dist archive. This is needed if # db-contrib-tool cannot autodetect the platform of the ec2 instance. regex='MONGO_DISTMOD=([a-z0-9]*)' - if [[ ${compile_flags} =~ ${regex} ]]; then + if [[ ${bazel_compile_flags} =~ ${regex} ]]; then extra_db_contrib_args="${extra_db_contrib_args} --platform=${BASH_REMATCH[1]}" fi download_dir="./tmp_db_contrib_tool_download_dir" rm -rf ${download_dir} - if [ "${task_name}" = "compile_dist_test" ]; then - file_name="mongodb-binaries.${ext}" + if [ "${task_name}" = "archive_dist_test" ]; then + file_name="dist-test-stripped.${ext}" invocation="db-contrib-tool setup-repro-env ${build_patch_id} \ --variant=${compile_variant} --extractDownloads=False \ --binariesName=${file_name} --installDir=${download_dir} ${extra_db_contrib_args}" fi if [ "${task_name}" = "archive_dist_test_debug" ]; then - file_name="mongo-debugsymbols.${ext}" + file_name="dist-test-debug.${ext}" invocation="db-contrib-tool setup-repro-env ${build_patch_id} \ --variant=${compile_variant} --extractDownloads=False \ --debugsymbolsName=${file_name} --installDir=${download_dir} \ @@ -71,7 +67,8 @@ if [ -n "${build_patch_id}" ]; then fi file_location=$(find "${download_dir}" -name "${file_name}") echo "Downloaded: ${file_location}" - mv "${file_location}" "${file_name}" + mkdir -p bazel-bin + mv "${file_location}" "bazel-bin/${file_name}" echo "Moved ${file_name} to the correct location" echo "Skipping ${task_name} compile" exit 0 @@ -85,6 +82,7 @@ fi if [ "${build_mongot}" = "true" ]; then setup_db_contrib_tool use_db_contrib_tool_mongot + bazel_args="${bazel_args} --include_mongot=True" fi set -o pipefail @@ -114,11 +112,33 @@ if [ -n "${build_timeout_seconds}" ]; then TIMEOUT_CMD="timeout ${build_timeout_seconds}" fi -for i in {1..5}; do - eval ${TIMEOUT_CMD} $BAZEL_BINARY build --verbose_failures $LOCAL_ARG ${args} ${targets} && RET=0 && break || RET=$? && sleep 1 +if is_ppc64le; then + LOCAL_ARG="$LOCAL_ARG --jobs=48" +fi + +if is_s390x; then + LOCAL_ARG="$LOCAL_ARG --jobs=16" +fi + +# If we are doing a patch build or we are building a non-push +# build on the waterfall, then we don't need the --release +# flag. Otherwise, this is potentially a build that "leaves +# the building", so we do want that flag. +if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then + echo "This is a non-release build." +else + LOCAL_ARG="$LOCAL_ARG --config=public-release" +fi + +for i in {1..3}; do + eval ${TIMEOUT_CMD} $BAZEL_BINARY build --verbose_failures $LOCAL_ARG ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \ + --define=MONGO_VERSION=${version} ${patch_compile_flags} ${targets} 2>&1 | tee bazel_stdout.log \ + && RET=0 && break || RET=$? && sleep 1 if [ $RET -eq 124 ]; then echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..." else + echo "Errors were found during the bazel run, here are the errors:" 1>&2 + grep "ERROR:" bazel_stdout.log 1>&2 echo "Bazel failed to execute, retrying..." fi done diff --git a/evergreen/bazel_get_binary_path.sh b/evergreen/bazel_get_binary_path.sh deleted file mode 100644 index 278350aefdd..00000000000 --- a/evergreen/bazel_get_binary_path.sh +++ /dev/null @@ -1,9 +0,0 @@ -bazel_get_binary_path() { - ARCH="$(uname -m)" - - if [[ "$ARCH" == "ppc64le" || "$ARCH" == "ppc64" || "$ARCH" == "ppc" || "$ARCH" == "ppcle" || "$ARCH" == "s390x" || "$ARCH" == "s390" ]]; then - return "bazel/bazelisk.py" - else - return "bazel" - fi -} diff --git a/evergreen/bazel_scons_diff.sh b/evergreen/bazel_scons_diff.sh deleted file mode 100644 index 766636d9061..00000000000 --- a/evergreen/bazel_scons_diff.sh +++ /dev/null @@ -1,41 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -cd src - -set -o errexit -set -o verbose - -ARCH=$(uname -m) -if [[ "$ARCH" == "arm64" || "$ARCH" == "aarch64" ]]; then - ARCH="arm64" -elif [[ "$ARCH" == "ppc64le" || "$ARCH" == "ppc64" || "$ARCH" == "ppc" || "$ARCH" == "ppcle" ]]; then - ARCH="ppc64le" -elif [[ "$ARCH" == "s390x" || "$ARCH" == "s390" ]]; then - ARCH="s390x" -else - ARCH="amd64" -fi - -bazel_binary=$TMPDIR/bazelisk -if [[ $ARCH == "ppc64le" ]] || [[ $ARCH == "s390x" ]]; then - bazel_binary=$TMPDIR/bazel -fi - -extra_args="--extra_args \"" -extra_args="$extra_args ${compile_flags}" -extra_args="$extra_args --evergreen-tmp-dir=${TMPDIR}" -extra_args="$extra_args\"" - -activate_venv - -if [[ -z "${bazel_scons_diff_targets}" ]]; then - echo "Skipping diff run since bazel_scons_diff_targets was not set" - exit 0 -fi - -eval ${compile_env} $python ./buildscripts/bazel_scons_diff.py \ - --bazel_binary ${bazel_binary} \ - ${extra_args} \ - ${bazel_scons_diff_targets} -exit $? diff --git a/evergreen/bazel_utility_functions.sh b/evergreen/bazel_utility_functions.sh index 568c3171552..dbe470c54cd 100755 --- a/evergreen/bazel_utility_functions.sh +++ b/evergreen/bazel_utility_functions.sh @@ -1,7 +1,25 @@ -is_s390x_or_ppc64le() { +is_ppc64le() { ARCH="$(uname -m)" - if [[ "$ARCH" == "ppc64le" || "$ARCH" == "ppc64" || "$ARCH" == "ppc" || "$ARCH" == "ppcle" || "$ARCH" == "s390x" || "$ARCH" == "s390" ]]; then + if [[ "$ARCH" == "ppc64le" || "$ARCH" == "ppc64" || "$ARCH" == "ppc" ]]; then + return 0 + else + return 1 + fi +} + +is_s390x() { + ARCH="$(uname -m)" + + if [[ "$ARCH" == "s390x" || "$ARCH" == "s390" ]]; then + return 0 + else + return 1 + fi +} + +is_s390x_or_ppc64le() { + if is_ppc64le || is_s390x; then return 0 else return 1 @@ -9,10 +27,12 @@ is_s390x_or_ppc64le() { } bazel_get_binary_path() { - ARCH="$(uname -m)" - if is_s390x_or_ppc64le; then echo "bazel/bazelisk.py" + elif grep -q "ID=debian" /etc/os-release; then + echo "bazel/bazelisk.py" + elif grep -q 'ID="sles"' /etc/os-release; then + echo "bazel/bazelisk.py" else echo "bazel" fi diff --git a/evergreen/crypt_run_tests.sh b/evergreen/crypt_run_tests.sh index 6224f4cd8c0..5f0f17aca98 100755 --- a/evergreen/crypt_run_tests.sh +++ b/evergreen/crypt_run_tests.sh @@ -16,7 +16,7 @@ if [ "$(uname)" != "Linux" ]; then exit 0 fi -EXTRACT_DIR="build/crypt-lib-${version}" +EXTRACT_DIR="bazel-bin/install" SOPATH="${EXTRACT_DIR}/lib/mongo_crypt_v1.so" UNITTEST_PATH="${EXTRACT_DIR}/bin/mongo_crypt_shlib_test" GDB_PATH="/opt/mongodbtoolchain/v5/bin/gdb" diff --git a/evergreen/failed_unittests_gather.sh b/evergreen/failed_unittests_gather.sh index 5908f08c33f..b63e64d66f3 100644 --- a/evergreen/failed_unittests_gather.sh +++ b/evergreen/failed_unittests_gather.sh @@ -80,7 +80,7 @@ while read -r core_file; do done <<< "${core_files}" # Copy debug symbols for dynamic builds -lib_dir=build/install/lib +lib_dir=bazel-bin/install/lib if [ -d "$lib_dir" ] && [ -n "$core_files" ]; then cp -r "$lib_dir" dist-unittests fi diff --git a/evergreen/functions/bolt.sh b/evergreen/functions/bolt.sh index a75a2d25c8e..dc1be824065 100755 --- a/evergreen/functions/bolt.sh +++ b/evergreen/functions/bolt.sh @@ -10,8 +10,8 @@ if [ -z "${BOLT:-}" ]; then exit 0 fi -tar -xvf mongodb-binaries.tgz -sudo rm mongodb-binaries.tgz +tar -xvf bazel-bin/dist-test-stripped.tgz +sudo rm bazel-bin/dist-test-stripped.tgz wget https://dsi-donot-remove.s3.us-west-2.amazonaws.com/llvm-bolt.gz wget https://dsi-donot-remove.s3.us-west-2.amazonaws.com/bolt.data.gz @@ -24,4 +24,4 @@ gunzip bolt.data.gz sudo rm dist-test/bin/mongod cp mongod_new dist-test/bin/mongod -tar -czvf mongodb-binaries.tgz dist-test +tar -czvf bazel-bin/dist-test-stripped.tgz dist-test diff --git a/evergreen/functions/compile_expansions_generate.sh b/evergreen/functions/compile_expansions_generate.sh deleted file mode 100644 index e6041a2cb8c..00000000000 --- a/evergreen/functions/compile_expansions_generate.sh +++ /dev/null @@ -1,52 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/../prelude.sh" - -cd src - -set -o errexit -set -o verbose -activate_venv -# shared scons cache testing -# if 'scons_cache_scope' enabled and project level 'disable_shared_scons_cache' is not true -# 'scons_cache_scope' is set on a per variant basis -# 'disable_shared_scons_cache' is set on a project level and applies to all variants -# Shared - if scons_cache_scope is set, then use new shared scons cache settings -if [ ! -z ${scons_cache_scope} ]; then - if [ "${disable_shared_scons_cache}" = "true" ]; then - echo "SCons Cache disabled. All shared scons settings will be ignored" - scons_cache_scope=none - else - scons_cache_scope=${scons_cache_scope} - fi - if [ "$scons_cache_scope" = "shared" ]; then - set +o errexit - if [ "Windows_NT" = "$OS" ]; then - ./win_mount.sh - else - mount | grep "\/efs" > /dev/null - if [ $? -eq 0 ]; then - echo "Shared cache is already mounted" - else - echo "Shared cache - mounting file system" - set_sudo - $sudo mount /efs - fi - fi - set -o errexit - fi - echo "Shared Cache with setting: ${scons_cache_scope}" - if [ ! -z ${commit_queue_alternate_cache} ]; then - SCONS_CACHE_DIR=${project}_${commit_queue_alternate_cache} - else - SCONS_CACHE_DIR=${project}_${build_variant} - fi - - SCONS_CACHE_DIR=$SCONS_CACHE_DIR SCONS_CACHE_MODE=${scons_cache_mode} SCONS_CACHE_SCOPE=$scons_cache_scope IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_compile_expansions_shared_cache.py --out compile_expansions.yml -# Legacy Expansion generation -else - echo "Using legacy expansion generation" - # Proceed with regular expansions generated - # This script converts the generated version string into a sanitized version string for - # use by scons and uploading artifacts as well as information about for the scons cache. - SCONS_CACHE_MODE=${scons_cache_mode} USE_SCONS_CACHE=${use_scons_cache} IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_compile_expansions.py --out compile_expansions.yml -fi diff --git a/evergreen/functions/fix_tar.sh b/evergreen/functions/fix_tar.sh new file mode 100755 index 00000000000..119ab9e5043 --- /dev/null +++ b/evergreen/functions/fix_tar.sh @@ -0,0 +1,16 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/../prelude.sh" + +set +o errexit + +cd src + +# For whatever reason, signing on mac doesn't like the tar file produced from bazel +# Round trip the tar by un-taring then re-taring it +if [[ "$OSTYPE" == "darwin"* ]]; then + mkdir ./tmp-tar + tar -xvf $1 -C ./tmp-tar + sudo rm $1 + tar -czvf $1 -C ./tmp-tar . + sudo rm -rf ./tmp-tar +fi diff --git a/evergreen/functions/shared_scons_directory_umount.sh b/evergreen/functions/shared_scons_directory_umount.sh deleted file mode 100755 index 06d0465161e..00000000000 --- a/evergreen/functions/shared_scons_directory_umount.sh +++ /dev/null @@ -1,19 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/../prelude.sh" - -cd src - -set -o errexit -set -o verbose - -if [ "${disable_shared_scons_cache}" = true ]; then - exit -fi -if [ "${scons_cache_scope}" = "shared" ]; then - if [ "Windows_NT" = "$OS" ]; then - net use X: /delete || true - else - set_sudo - $sudo umount /efs || true - fi -fi diff --git a/evergreen/generate_evergreen_bazelrc.sh b/evergreen/generate_evergreen_bazelrc.sh index 4ec4431c685..3421553246f 100644 --- a/evergreen/generate_evergreen_bazelrc.sh +++ b/evergreen/generate_evergreen_bazelrc.sh @@ -11,8 +11,6 @@ mkdir -p $TMPDIR if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then mkdir -p Z:/bazel_tmp touch Z:/bazel_tmp/mci_path - echo "--action_env=TMP=Z:/bazel_tmp" >> .bazelrc.evergreen - echo "--action_env=TEMP=Z:/bazel_tmp" >> .bazelrc.evergreen # TODO(SERVER-94605): remove when Windows temp directory is cleared between task runs if [[ "$PWD" != "$(cat Z:/bazel_tmp/mci_path)" ]]; then echo "Clearing bazel output root from previous task mci '$(cat Z:/bazel_tmp/mci_path)'" @@ -24,8 +22,11 @@ if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then # see https://jira.mongodb.org/browse/DEVPROD-11126 abs_path=$(cygpath -w "$TMPDIR" | tr '\\' '/') echo "startup --output_user_root=Z:/bazel_tmp" > .bazelrc.evergreen + echo "common --action_env=TMP=Z:/bazel_tmp" >> .bazelrc.evergreen + echo "common --action_env=TEMP=Z:/bazel_tmp" >> .bazelrc.evergreen echo "BAZELISK_HOME=${abs_path}/bazelisk_home" >> .bazeliskrc - echo "common --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" >> .bazelrc.git + # echo "common --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" >> .bazelrc.git + echo "common --define GIT_COMMIT_HASH=nogitversion" >> .bazelrc.git else echo "startup --output_user_root=${TMPDIR}/bazel-output-root" > .bazelrc.evergreen echo "BAZELISK_HOME=${TMPDIR}/bazelisk_home" >> .bazeliskrc diff --git a/evergreen/packager.py_run.sh b/evergreen/packager.py_run.sh index f024abfa3b0..ea71996e701 100644 --- a/evergreen/packager.py_run.sh +++ b/evergreen/packager.py_run.sh @@ -20,5 +20,5 @@ if [ -z ${packager_script+x} ]; then fi cd buildscripts -$python ${packager_script} --prefix $(pwd)/.. --distros ${packager_distro} --tarball $(pwd)/../mongodb-dist.tgz -s ${version} -m HEAD -a ${packager_arch} +$python ${packager_script} --prefix $(pwd)/.. --distros ${packager_distro} --tarball $(pwd)/../bazel-bin/dist-stripped.tgz -s ${version} -m HEAD -a ${packager_arch} cd .. diff --git a/evergreen/prelude_db_contrib_tool.sh b/evergreen/prelude_db_contrib_tool.sh index b82e4eaad76..ad92397cbd9 100755 --- a/evergreen/prelude_db_contrib_tool.sh +++ b/evergreen/prelude_db_contrib_tool.sh @@ -58,4 +58,6 @@ function use_db_contrib_tool_mongot { # This should create the folder mongot-localdev, usually run at the root of mongo directory curl ${mongot_url} | tar xvz fi + # Hack to remove BUILD.bazel file that can be lying around in mongot + rm -f ./mongot-localdev/bin/jdk/BUILD.bazel } diff --git a/evergreen/scons_compile.sh b/evergreen/scons_compile.sh deleted file mode 100755 index 0b8c806a473..00000000000 --- a/evergreen/scons_compile.sh +++ /dev/null @@ -1,196 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -cd src - -set -o errexit -set -o verbose - -rm -rf ${install_directory} - -# By default, limit link jobs to one quarter of our overall -j -# concurrency unless locally overridden. We do this because in -# static link environments, the memory consumption of each -# link job is so high that without constraining the number of -# links we are likely to OOM or thrash the machine. Dynamic -# builds, where htis is not a concern, override this value. -echo "Changing SCons to run with --jlink=${num_scons_link_jobs_available}" -extra_args="$extra_args --jlink=${num_scons_link_jobs_available} --separate-debug=${separate_debug} --skip-archive=${skip_archive}" - -echo "Changing SCons to run with UNITTESTS_COMPILE_CONCURRENCY=${num_scons_unit_cc_jobs_available}" -extra_args="$extra_args UNITTESTS_COMPILE_CONCURRENCY=${num_scons_unit_cc_jobs_available}" - -extra_args="$extra_args --consolidated-test-bins=on" - -if [ "${scons_cache_scope}" = "shared" ]; then - extra_args="$extra_args --cache-debug=scons_cache.log" -fi - -# Conditionally enable scons time debugging -if [ "${show_scons_timings}" = "true" ]; then - extra_args="$extra_args --debug=time,memory,count" -fi - -# Build packages where the upload tasks expect them -if [ -n "${git_project_directory}" ]; then - extra_args="$extra_args PKGDIR='${git_project_directory}'" -else - extra_args="$extra_args PKGDIR='${workdir}/src'" -fi - -# If we are doing a patch build or we are building a non-push -# build on the waterfall, then we don't need the --release -# flag. Otherwise, this is potentially a build that "leaves -# the building", so we do want that flag. The non --release -# case should auto enale the faster decider when -# applicable. Furthermore, for the non --release cases we can -# accelerate the build slightly for situations where we invoke -# SCons multiple times on the same machine by allowing SCons -# to assume that implicit dependencies are cacheable across -# runs. -if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then - extra_args="$extra_args --implicit-cache --build-fast-and-loose=on" -else - extra_args="$extra_args --release" -fi - -extra_args="$extra_args SPLIT_DWARF=0 GDB_INDEX=0 ENABLE_BUILD_RETRY=1 BAZEL_INTEGRATION_DEBUG=1" - -echo "Setting evergreen tmp dir to $TMPDIR" -extra_args="$extra_args --evergreen-tmp-dir='${TMPDIR}'" - -if [ "${generating_for_ninja}" = "true" ] && [ "Windows_NT" = "$OS" ]; then - vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/" - export PATH="$(echo "$(cd "$vcvars" && cmd /C "vcvarsall.bat amd64 && C:/cygwin/bin/bash -c 'echo \$PATH'")" | tail -n +6)":$PATH -fi -activate_venv - -# if build_patch_id is passed, try to download binaries from specified -# evergreen patch. -# This is purposfully before the venv setup so we do not touch the venv deps -build_patch_id="${build_patch_id:-${reuse_compile_from}}" -if [ -n "${build_patch_id}" ]; then - echo "build_patch_id detected, trying to skip task" - if [ "${task_name}" = "compile_dist_test" ] || [ "${task_name}" = "compile_dist_test_half" ]; then - echo "Skipping ${task_name} compile without downloading any files" - exit 0 - fi - - # On windows we change the extension to zip - if [ -z "${ext}" ]; then - ext="tgz" - fi - - extra_db_contrib_args="" - - # get the platform of the dist archive. This is needed if - # db-contrib-tool cannot autodetect the platform of the ec2 instance. - regex='MONGO_DISTMOD=([a-z0-9]*)' - if [[ ${compile_flags} =~ ${regex} ]]; then - extra_db_contrib_args="${extra_db_contrib_args} --platform=${BASH_REMATCH[1]}" - fi - - download_dir="./tmp_db_contrib_tool_download_dir" - rm -rf ${download_dir} - - if [ "${task_name}" = "archive_dist_test" ]; then - file_name="mongodb-binaries.${ext}" - invocation="db-contrib-tool setup-repro-env ${build_patch_id} \ - --variant=${compile_variant} --extractDownloads=False \ - --binariesName=${file_name} --installDir=${download_dir} ${extra_db_contrib_args}" - fi - - if [ "${task_name}" = "archive_dist_test_debug" ]; then - file_name="mongo-debugsymbols.${ext}" - invocation="db-contrib-tool setup-repro-env ${build_patch_id} \ - --variant=${compile_variant} --extractDownloads=False \ - --debugsymbolsName=${file_name} --installDir=${download_dir} \ - --skipBinaries --downloadSymbols ${extra_db_contrib_args}" - fi - - if [ -n "${invocation}" ]; then - setup_db_contrib_tool - - echo "db-contrib-tool invocation: ${invocation}" - eval ${invocation} - if [ $? -ne 0 ]; then - echo "Could not retrieve files with db-contrib-tool" - exit 1 - fi - file_location=$(find "${download_dir}" -name "${file_name}") - echo "Downloaded: ${file_location}" - mv "${file_location}" "${file_name}" - echo "Moved ${file_name} to the correct location" - echo "Skipping ${task_name} compile" - exit 0 - fi - - echo "Could not skip ${task_name} compile, compiling as normal" -fi - -# --build-mongot is a compile flag used by the evergreen build variants that run end-to-end search -# suites, as it downloads the necessary mongot binary. -if [ "${build_mongot}" = "true" ]; then - # Checking that this is not a downstream patch on mongod created by mongot's patch trigger. - # In the case that it's not, download latest (eg HEAD of 10gen/mongot) or the - # release (eg currently running in production on Atlas) mongot binary. - if [[ ! $(declare -p linux_x86_64_mongot_localdev_binary linux_aarch64_mongot_localdev_binary macos_x86_64_mongot_localdev_binary 2> /dev/null) ]]; then - if [ "${download_mongot_release}" = "true" ]; then - extra_args="$extra_args --build-mongot=release" - else - extra_args="$extra_args --build-mongot=latest" - fi - else - # This is a downstream patch, which means there is a patched mongot binary we need to install. - if [[ "$OSTYPE" == "linux-gnu"* ]]; then - arch=$(uname -i) - if [[ $arch == x86_64* ]]; then - extra_args="$extra_args --patch-build-mongot-url=${linux_x86_64_mongot_localdev_binary}" - elif [[ $arch == aarch64* ]]; then - extra_args="$extra_args --patch-build-mongot-url=${linux_aarch64_mongot_localdev_binary}" - else - echo "mongot-localdev does not support ${arch}" - exit 1 - fi - elif [[ "$OSTYPE" == "darwin"* ]]; then - extra_args="$extra_args --patch-build-mongot-url=${macos_x86_64_mongot_localdev_binary}" - else - echo "mongot-localdev does not support ${OSTYPE}" - exit 1 - fi - fi -fi - -set -o pipefail - -# Bind mount a new tmp directory to the real /tmp to circumvent "out of disk space" errors on ARM LTO compiles -# the /tmp directory is limited to 32 GB while the home directory has 500 GB -if [[ ${compile_flags} == *"--lto"* ]]; then - set_sudo - mkdir -p tmp && $sudo mount --bind ./tmp /tmp -fi - -if [[ -n "${bazel_scons_diff_targets}" ]]; then - diff_extra_args="--extra_args \"${compile_flags} ${task_compile_flags} ${task_compile_flags_extra} $extra_args\"" - eval ${compile_env} $python ./buildscripts/bazel_scons_diff.py \ - ${diff_extra_args} \ - ${bazel_scons_diff_targets} -fi - -if [[ -n "${convert_bazel_headers_target}" ]]; then - eval ${compile_env} $python ./buildscripts/convert_bazel_headers.py --target-library=${convert_bazel_headers_target} --silent > bazel_headers.txt - exit 1 -fi - -eval ${compile_env} $python ./buildscripts/scons.py \ - ${compile_flags} ${task_compile_flags} ${bazel_build_tags} ${task_compile_flags_extra} \ - ${scons_cache_args} $extra_args \ - ${targets} MONGO_VERSION=${version} ${patch_compile_flags} | tee scons_stdout.log -exit_status=$? - -# If compile fails we do not run any tests -if [[ $exit_status -ne 0 ]]; then - touch ${skip_tests} -fi - -exit $exit_status diff --git a/evergreen/scons_raw_compile.sh b/evergreen/scons_raw_compile.sh deleted file mode 100755 index 7e9de157f98..00000000000 --- a/evergreen/scons_raw_compile.sh +++ /dev/null @@ -1,26 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -# Compile without extra args from evergreen to better simulate local dev compilation. - -cd src - -set -o errexit -set -o verbose - -if [ "Windows_NT" = "$OS" ]; then - vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/" - export PATH="$(echo "$(cd "$vcvars" && cmd /C "vcvarsall.bat amd64 && C:/cygwin/bin/bash -c 'echo \$PATH'")" | tail -n +6)":$PATH -fi -activate_venv - -set -o pipefail - -echo "Setting evergreen tmp dir to $TMPDIR" -compile_flags="$compile_flags --evergreen-tmp-dir='${TMPDIR}'" - -eval ${compile_env} $python ./buildscripts/scons.py \ - ${compile_flags} ${task_compile_flags} \ - ${targets} | tee scons_stdout.log - -exit $? diff --git a/evergreen/scons_splunk.sh b/evergreen/scons_splunk.sh deleted file mode 100644 index 08b76ffe873..00000000000 --- a/evergreen/scons_splunk.sh +++ /dev/null @@ -1,6 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -touch src/scons_cache.log.json -# TODO EVG-18207 remove GODEBUG=asyncpreemptoff=1 -GODEBUG=asyncpreemptoff=1 curator --level warning splunk --json --url=${scons_splunk_server} --token=${scons_splunk_token} --annotation=project:${project} --annotation=task_id:${task_id} --annotation=build_variant:${build_variant} --annotation=git_revision:${revision} follow --file src/scons_cache.log.json diff --git a/evergreen/stitch_support_create_lib_tar.sh b/evergreen/stitch_support_create_lib_tar.sh deleted file mode 100755 index e5ddb1e2524..00000000000 --- a/evergreen/stitch_support_create_lib_tar.sh +++ /dev/null @@ -1,9 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -cd src/build - -set -o errexit -set -o verbose - -tar cfvz stitch-support.tgz stitch-support-lib-${version} diff --git a/evergreen/stitch_support_run_tests.sh b/evergreen/stitch_support_run_tests.sh index d7b69d01679..63390093e44 100755 --- a/evergreen/stitch_support_run_tests.sh +++ b/evergreen/stitch_support_run_tests.sh @@ -6,4 +6,4 @@ cd src set -o errexit set -o verbose -"build/stitch-support-lib-${version}/bin/stitch_support_test" +"bazel-bin/install/bin/stitch_support_test" diff --git a/jstests/libs/hang_test_to_attach_gdb.js b/jstests/libs/hang_test_to_attach_gdb.js index 5560696e697..fc76b72a397 100644 --- a/jstests/libs/hang_test_to_attach_gdb.js +++ b/jstests/libs/hang_test_to_attach_gdb.js @@ -27,7 +27,7 @@ function isTimeToWake(waitFor) { * * Once you have your test hanging, you can use the 'ps' tool to find the process id (pid) of the * target node, and then do something like: - * /opt/mongodbtoolchain/v4/bin/gdb build/install/bin/mongod + * /opt/mongodbtoolchain/v4/bin/gdb bazel-bin/install/bin/mongod * ... * (gdb prompt) attach * diff --git a/src/mongo/embedded/stitch_support/BUILD.bazel b/src/mongo/embedded/stitch_support/BUILD.bazel index 59d0740ec21..6f2f83ac23c 100644 --- a/src/mongo/embedded/stitch_support/BUILD.bazel +++ b/src/mongo/embedded/stitch_support/BUILD.bazel @@ -9,6 +9,7 @@ exports_files( ]), ) +# make sure to modify stitch_support_for_test if you modify this definition mongo_cc_library( name = "stitch_support", srcs = [ @@ -33,6 +34,71 @@ mongo_cc_library( copts = [ "-Isrc/mongo/embedded", ], + linkshared = True, + local_defines = [ + "STITCH_SUPPORT_COMPILING", + ] + select({ + "//bazel/config:linkstatic_enabled": [ + "STITCH_SUPPORT_STATIC", + ], + "//conditions:default": [], + }), + non_transitive_dyn_linkopts = select({ + "@platforms//os:linux": [ + "-Wl,--version-script=$(location :libstitch_support.version_script.lds)", + ], + "@platforms//os:macos": [ + "-Wl,-exported_symbols_list,$(location :libstitch_support.exported_symbols_list.lds)", + ], + "@platforms//os:windows": [ + "/DEF:$(location :stitch_support.def)", + ], + "//conditions:default": [], + }), + shared_lib_name = select({ + "@platforms//os:windows": "stitch_support_shared_with_debug.dll", + "//conditions:default": "stitch_support_shared_with_debug.so", + }), + tags = ["stitch_support"], + target_compatible_with = select({ + "//bazel/config:shared_archive_or_link_dynamic": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + deps = [ + "//src/mongo/db:query_expressions", + "//src/mongo/db:vector_clock_trivial", + "//src/mongo/db/index:index_access_method", + "//src/mongo/db/query/collation:collator_factory_icu", + "//src/mongo/db/query/collation:collator_factory_interface", + "//src/mongo/db/query/write_ops:parsed_update", + ], +) + +# This is a duplicate definition so we can build a regular .so file for testing +mongo_cc_library( + name = "stitch_support_duplicate", + srcs = [ + "stitch_support.cpp", + ], + hdrs = [ + "stitch_support.h", + "//src/mongo/embedded:api_common.h", + ], + additional_linker_inputs = select({ + "@platforms//os:linux": [ + ":libstitch_support.version_script.lds", + ], + "@platforms//os:macos": [ + ":libstitch_support.exported_symbols_list.lds", + ], + "@platforms//os:windows": [ + ":stitch_support.def", + ], + "//conditions:default": [], + }), + copts = [ + "-Isrc/mongo/embedded", + ], local_defines = [ "STITCH_SUPPORT_COMPILING", ] + select({ @@ -82,7 +148,7 @@ mongo_cc_unit_test( "//conditions:default": [], }), deps = [ - "stitch_support", + "stitch_support_duplicate", "//src/mongo/db:service_context_non_d", "//src/mongo/unittest", ], diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 1cb377edf1d..2d6192c5c0b 100644 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -78,7 +78,7 @@ functions: else export PATH=/opt/mongodbtoolchain/v4/bin:$PATH fi - # Since Bazel (currently used in SCons) uses EngFlow's remote execution system instead of icecream, + # Since Bazel uses EngFlow's remote execution system # additional credentials need to be setup to maintain efficient compilation speed. "get engflow creds": &get_engflow_creds - command: s3.get