SERVER-110392 Collect coredumps in unit tests during remote execution (#41130)

GitOrigin-RevId: 5b7cd5640b074c12e26702adf9f89316a7eeec47

bazel/BUILD.bazel

@@ -33,3 +33,8 @@ py_binary(
         ),
     ],
 )
+
+filegroup(
+    name = "test_wrapper",
+    srcs = ["test_wrapper.sh"],
+)

bazel/mongo_src_rules.bzl

@@ -852,6 +852,22 @@ def _mongo_cc_binary_and_test(
             exec_properties = exec_properties,
         )
 
+        native.sh_test(
+            name = name + "_remote_exec",
+            srcs = [
+                "//bazel:test_wrapper",
+            ],
+            tags = original_tags + ["remote_exec_test"],
+            args = [
+                "$(location " + name + ")",
+            ],
+            data = args["data"] + [name],
+            env = args["env"],
+            target_compatible_with = target_compatible_with,
+            visibility = visibility,
+            exec_properties = exec_properties,
+        )
+
 def mongo_cc_binary(
         name,
         srcs = [],

bazel/platforms/remote_execution_containers_generator.py

@@ -7,7 +7,10 @@ import pathlib
 import subprocess
 from datetime import datetime
 
+from retry import retry
 
+
+@retry(tries=3)
 def log_subprocess_run(*args, **kwargs):
     arg_list_or_string = kwargs["args"] if "args" in kwargs else args[0]
     print(" ".join(arg_list_or_string) if type(arg_list_or_string) == list else arg_list_or_string)

bazel/test_wrapper.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+ulimit -c unlimited
+
+$1
+RET=$?
+
+CORE_FILE=$(find -L ./ -name "core")
+if [ -f "$CORE_FILE" ]; then
+    CORE_FILENAME="dump_$(date +%s%N).core"
+    mv $CORE_FILE "$TEST_UNDECLARED_OUTPUTS_DIR/$CORE_FILENAME"
+    echo "Writing coredump to $CORE_FILENAME..."
+fi
+
+exit $RET

buildscripts/create_bazel_test_report.py

@@ -38,7 +38,9 @@ def main(testlog_dir: str):
     for test_xml in glob(f"{testlog_dir}/**/test.xml", recursive=True):
         testsuite = ET.parse(test_xml).getroot().find("testsuite")
         testcase = testsuite.find("testcase")
-        test_file = testcase.attrib["name"]
+
+        # Replace part of the name added by the remote test wrapper script
+        test_file = testcase.attrib["name"].replace("_remote_exec", "")
 
         if testcase.find("error") is not None:
             status = "fail"

etc/evergreen_yml_components/tasks/misc_tasks.yml

@@ -1029,7 +1029,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --linkstatic=False
-            --test_tag_filters=mongo_unittest,-intermediate_debug,-incompatible_with_bazel_remote_test
+            --test_tag_filters=mongo_unittest,-final_target,-intermediate_debug,-incompatible_with_bazel_remote_test
             --build_tag_filters=mongo_unittest,-incompatible_with_bazel_remote_test
 
   - name: unit_tests_remote_exec_incompatible
@@ -1054,7 +1054,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=incompatible_with_bazel_remote_test,-intermediate_debug
+            --test_tag_filters=incompatible_with_bazel_remote_test,-intermediate_debug,-remote_exec_test
             --build_tag_filters=incompatible_with_bazel_remote_test
 
   - name: unit_test_group1
@@ -1080,7 +1080,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_first_group,-intermediate_debug
+            --test_tag_filters=mongo_unittest_first_group,-intermediate_debug,-remote_exec_test
             --build_tag_filters=mongo_unittest_first_group
 
   - name: unit_test_group2
@@ -1106,7 +1106,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_second_group,-intermediate_debug,-tracing_test
+            --test_tag_filters=mongo_unittest_second_group,-intermediate_debug,-tracing_test,-remote_exec_test
             --build_tag_filters=mongo_unittest_second_group
 
   - name: unit_test_group3
@@ -1132,7 +1132,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_third_group,-intermediate_debug
+            --test_tag_filters=mongo_unittest_third_group,-intermediate_debug,-remote_exec_test
             --build_tag_filters=mongo_unittest_third_group
 
   - name: unit_test_group4
@@ -1158,7 +1158,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_fourth_group,-intermediate_debug
+            --test_tag_filters=mongo_unittest_fourth_group,-intermediate_debug,-remote_exec_test
             --build_tag_filters=mongo_unittest_fourth_group
 
   - name: unit_test_group5
@@ -1184,7 +1184,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_fifth_group,-intermediate_debug
+            --test_tag_filters=mongo_unittest_fifth_group,-intermediate_debug,-remote_exec_test
             --build_tag_filters=mongo_unittest_fifth_group
 
   - name: unit_test_group6
@@ -1210,7 +1210,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_sixth_group,-intermediate_debug
+            --test_tag_filters=mongo_unittest_sixth_group,-intermediate_debug,-remote_exec_test
             --build_tag_filters=mongo_unittest_sixth_group
 
   - name: unit_test_group7
@@ -1236,7 +1236,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_seventh_group,-intermediate_debug
+            --test_tag_filters=mongo_unittest_seventh_group,-intermediate_debug,-remote_exec_test
             --build_tag_filters=mongo_unittest_seventh_group
 
   - name: unit_test_group8
@@ -1262,7 +1262,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_eighth_group,-intermediate_debug
+            --test_tag_filters=mongo_unittest_eighth_group,-intermediate_debug,-remote_exec_test
             --build_tag_filters=mongo_unittest_eighth_group
 
   - name: monitor_build_status

etc/evergreen_yml_components/variants/sanitizer/test_dev.yml

@@ -499,6 +499,8 @@ buildvariants:
     tasks:
       - name: compile_test_parallel_core_stream_TG
       - name: run_unit_tests_TG
+        distros:
+          - rhel8.8-xlarge
       - name: compile_test_parallel_dbtest_stream_TG
       - name: compile_integration_and_test_parallel_stream_TG
       - name: compile_jstestshell_TG