From d0df302192d324f1305fa4cedd09722d82ccd8ba Mon Sep 17 00:00:00 2001 From: Zack Winter <3457246+zackwintermdb@users.noreply.github.com> Date: Tue, 9 Sep 2025 19:44:06 -0700 Subject: [PATCH] SERVER-110392 Collect coredumps in unit tests during remote execution (#41130) GitOrigin-RevId: 5b7cd5640b074c12e26702adf9f89316a7eeec47 --- bazel/BUILD.bazel | 5 +++++ bazel/mongo_src_rules.bzl | 16 +++++++++++++++ .../remote_execution_containers_generator.py | 3 +++ bazel/test_wrapper.sh | 15 ++++++++++++++ buildscripts/create_bazel_test_report.py | 4 +++- .../tasks/misc_tasks.yml | 20 +++++++++---------- .../variants/sanitizer/test_dev.yml | 2 ++ 7 files changed, 54 insertions(+), 11 deletions(-) create mode 100755 bazel/test_wrapper.sh diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index 896fa3e107f..0b4b39015dc 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -33,3 +33,8 @@ py_binary( ), ], ) + +filegroup( + name = "test_wrapper", + srcs = ["test_wrapper.sh"], +) diff --git a/bazel/mongo_src_rules.bzl b/bazel/mongo_src_rules.bzl index 6af7e971224..0462e1944c7 100644 --- a/bazel/mongo_src_rules.bzl +++ b/bazel/mongo_src_rules.bzl @@ -852,6 +852,22 @@ def _mongo_cc_binary_and_test( exec_properties = exec_properties, ) + native.sh_test( + name = name + "_remote_exec", + srcs = [ + "//bazel:test_wrapper", + ], + tags = original_tags + ["remote_exec_test"], + args = [ + "$(location " + name + ")", + ], + data = args["data"] + [name], + env = args["env"], + target_compatible_with = target_compatible_with, + visibility = visibility, + exec_properties = exec_properties, + ) + def mongo_cc_binary( name, srcs = [], diff --git a/bazel/platforms/remote_execution_containers_generator.py b/bazel/platforms/remote_execution_containers_generator.py index 31d563f2d96..ad4f9634d3d 100755 --- a/bazel/platforms/remote_execution_containers_generator.py +++ b/bazel/platforms/remote_execution_containers_generator.py @@ -7,7 +7,10 @@ import pathlib import subprocess from datetime import datetime +from retry import retry + +@retry(tries=3) def log_subprocess_run(*args, **kwargs): arg_list_or_string = kwargs["args"] if "args" in kwargs else args[0] print(" ".join(arg_list_or_string) if type(arg_list_or_string) == list else arg_list_or_string) diff --git a/bazel/test_wrapper.sh b/bazel/test_wrapper.sh new file mode 100755 index 00000000000..a9f5c882ad1 --- /dev/null +++ b/bazel/test_wrapper.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +ulimit -c unlimited + +$1 +RET=$? + +CORE_FILE=$(find -L ./ -name "core") +if [ -f "$CORE_FILE" ]; then + CORE_FILENAME="dump_$(date +%s%N).core" + mv $CORE_FILE "$TEST_UNDECLARED_OUTPUTS_DIR/$CORE_FILENAME" + echo "Writing coredump to $CORE_FILENAME..." +fi + +exit $RET diff --git a/buildscripts/create_bazel_test_report.py b/buildscripts/create_bazel_test_report.py index 174c26f125b..54b078de529 100644 --- a/buildscripts/create_bazel_test_report.py +++ b/buildscripts/create_bazel_test_report.py @@ -38,7 +38,9 @@ def main(testlog_dir: str): for test_xml in glob(f"{testlog_dir}/**/test.xml", recursive=True): testsuite = ET.parse(test_xml).getroot().find("testsuite") testcase = testsuite.find("testcase") - test_file = testcase.attrib["name"] + + # Replace part of the name added by the remote test wrapper script + test_file = testcase.attrib["name"].replace("_remote_exec", "") if testcase.find("error") is not None: status = "fail" diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index 00dce2ad9d1..caa02d34877 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -1029,7 +1029,7 @@ tasks: bazel_args: >- --config=evg --linkstatic=False - --test_tag_filters=mongo_unittest,-intermediate_debug,-incompatible_with_bazel_remote_test + --test_tag_filters=mongo_unittest,-final_target,-intermediate_debug,-incompatible_with_bazel_remote_test --build_tag_filters=mongo_unittest,-incompatible_with_bazel_remote_test - name: unit_tests_remote_exec_incompatible @@ -1054,7 +1054,7 @@ tasks: bazel_args: >- --config=evg --include_autogenerated_targets=True - --test_tag_filters=incompatible_with_bazel_remote_test,-intermediate_debug + --test_tag_filters=incompatible_with_bazel_remote_test,-intermediate_debug,-remote_exec_test --build_tag_filters=incompatible_with_bazel_remote_test - name: unit_test_group1 @@ -1080,7 +1080,7 @@ tasks: bazel_args: >- --config=evg --include_autogenerated_targets=True - --test_tag_filters=mongo_unittest_first_group,-intermediate_debug + --test_tag_filters=mongo_unittest_first_group,-intermediate_debug,-remote_exec_test --build_tag_filters=mongo_unittest_first_group - name: unit_test_group2 @@ -1106,7 +1106,7 @@ tasks: bazel_args: >- --config=evg --include_autogenerated_targets=True - --test_tag_filters=mongo_unittest_second_group,-intermediate_debug,-tracing_test + --test_tag_filters=mongo_unittest_second_group,-intermediate_debug,-tracing_test,-remote_exec_test --build_tag_filters=mongo_unittest_second_group - name: unit_test_group3 @@ -1132,7 +1132,7 @@ tasks: bazel_args: >- --config=evg --include_autogenerated_targets=True - --test_tag_filters=mongo_unittest_third_group,-intermediate_debug + --test_tag_filters=mongo_unittest_third_group,-intermediate_debug,-remote_exec_test --build_tag_filters=mongo_unittest_third_group - name: unit_test_group4 @@ -1158,7 +1158,7 @@ tasks: bazel_args: >- --config=evg --include_autogenerated_targets=True - --test_tag_filters=mongo_unittest_fourth_group,-intermediate_debug + --test_tag_filters=mongo_unittest_fourth_group,-intermediate_debug,-remote_exec_test --build_tag_filters=mongo_unittest_fourth_group - name: unit_test_group5 @@ -1184,7 +1184,7 @@ tasks: bazel_args: >- --config=evg --include_autogenerated_targets=True - --test_tag_filters=mongo_unittest_fifth_group,-intermediate_debug + --test_tag_filters=mongo_unittest_fifth_group,-intermediate_debug,-remote_exec_test --build_tag_filters=mongo_unittest_fifth_group - name: unit_test_group6 @@ -1210,7 +1210,7 @@ tasks: bazel_args: >- --config=evg --include_autogenerated_targets=True - --test_tag_filters=mongo_unittest_sixth_group,-intermediate_debug + --test_tag_filters=mongo_unittest_sixth_group,-intermediate_debug,-remote_exec_test --build_tag_filters=mongo_unittest_sixth_group - name: unit_test_group7 @@ -1236,7 +1236,7 @@ tasks: bazel_args: >- --config=evg --include_autogenerated_targets=True - --test_tag_filters=mongo_unittest_seventh_group,-intermediate_debug + --test_tag_filters=mongo_unittest_seventh_group,-intermediate_debug,-remote_exec_test --build_tag_filters=mongo_unittest_seventh_group - name: unit_test_group8 @@ -1262,7 +1262,7 @@ tasks: bazel_args: >- --config=evg --include_autogenerated_targets=True - --test_tag_filters=mongo_unittest_eighth_group,-intermediate_debug + --test_tag_filters=mongo_unittest_eighth_group,-intermediate_debug,-remote_exec_test --build_tag_filters=mongo_unittest_eighth_group - name: monitor_build_status diff --git a/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml b/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml index 6b28adf3919..f8dd53567f9 100644 --- a/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml +++ b/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml @@ -499,6 +499,8 @@ buildvariants: tasks: - name: compile_test_parallel_core_stream_TG - name: run_unit_tests_TG + distros: + - rhel8.8-xlarge - name: compile_test_parallel_dbtest_stream_TG - name: compile_integration_and_test_parallel_stream_TG - name: compile_jstestshell_TG