SERVER-110647 Set up coredump collection logic on unit test timeout (#42592)

GitOrigin-RevId: 9f613d547669fd4e0708fb082a7838ea283947be

.bazelrc

@@ -414,7 +414,7 @@ common:remote_test --remote_download_outputs=minimal
 common:remote_test --test_output=summary
 common:remote_test --modify_execution_info=^(CppLink|CppArchive|SolibSymlink|ExtractDebuginfo|StripDebuginfo|CcGenerateIntermediateDwp|CcGenerateDwp)$=-no-remote-cache
 common:remote_test --remote_download_regex=.*(TestLogs.*|report.*\.json$|\.core$|data_archives.*\.tgz$) # Resmoke TestLogs directory, report, core dumps, and data directory archives.
-test:remote_test --test_timeout=600
+test:remote_test --test_timeout=660 # Allow extra 60s for coredump on abort
 test:remote_test --test_tag_filters=-incompatible_with_bazel_remote_test
 
 # Coverage

bazel/mongo_src_rules.bzl

@@ -854,11 +854,11 @@ def _mongo_cc_binary_and_test(
         )
 
         native.sh_test(
-            name = name + "_remote_exec",
+            name = name + "_ci_wrapper",
             srcs = [
                 "//bazel:test_wrapper",
             ],
-            tags = original_tags + ["remote_exec_test"],
+            tags = original_tags + ["wrapper_target"],
             args = [
                 "$(location " + name + ")",
             ],

bazel/test_wrapper.sh

@@ -2,7 +2,21 @@
 
 ulimit -c unlimited
 
-$1
+eval ${@:1} &
+main_pid=$!
+echo "Process-under-test started with PID: ${main_pid}"
+
+(
+    sleep 600
+
+    # 'kill -0' checks for process existence without sending a signal
+    if kill -0 "$main_pid" 2>/dev/null; then
+        echo "10 minute Timer finished. Process-under-test ${main_pid} is still running. Sending a SIGABRT to trigger a coredump now."
+        kill -ABRT "${main_pid}"
+    fi
+) &
+
+wait "${main_pid}"
 RET=$?
 
 CORE_FILE=$(find -L ./ -name "*.core")

buildscripts/create_bazel_test_report.py

@@ -41,7 +41,7 @@ def main(testlog_dir: str):
         testcase = testsuite.find("testcase")
 
         # Replace part of the name added by the remote test wrapper script
-        test_file = testcase.attrib["name"].replace("_remote_exec", "")
+        test_file = testcase.attrib["name"].replace("_ci_wrapper", "")
 
         if testcase.find("error") is not None:
             status = "fail"

buildscripts/gather_failed_unittests.py

@@ -59,9 +59,9 @@ def _copy_bins_to_upload(failed_tests: List[str], upload_bin_dir: str, upload_li
     # Search both in the top level remote exec shellscript wrapper output directory, and in the
     # binary output directory.
     failed_tests += [
-        failed_test.replace("_remote_exec", "")
+        failed_test.replace("_ci_wrapper", "")
         for failed_test in failed_tests
-        if "_remote_exec" in failed_test
+        if "_ci_wrapper" in failed_test
     ]
     for failed_test in failed_tests:
         full_binary_path = bazel_bin_dir / failed_test

etc/evergreen_yml_components/tasks/misc_tasks.yml

@@ -1068,7 +1068,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=incompatible_with_bazel_remote_test,-intermediate_debug,-remote_exec_test
+            --test_tag_filters=incompatible_with_bazel_remote_test,-intermediate_debug,-final_target
             --build_tag_filters=incompatible_with_bazel_remote_test
 
   - name: unit_test_group1
@@ -1094,7 +1094,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_first_group,-intermediate_debug,-remote_exec_test
+            --test_tag_filters=mongo_unittest_first_group,-intermediate_debug,-final_target
             --build_tag_filters=mongo_unittest_first_group
 
   - name: unit_test_group2
@@ -1120,7 +1120,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_second_group,-intermediate_debug,-tracing_test,-remote_exec_test
+            --test_tag_filters=mongo_unittest_second_group,-intermediate_debug,-tracing_test,-final_target
             --build_tag_filters=mongo_unittest_second_group
 
   - name: unit_test_group3
@@ -1146,7 +1146,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_third_group,-intermediate_debug,-remote_exec_test
+            --test_tag_filters=mongo_unittest_third_group,-intermediate_debug,-final_target
             --build_tag_filters=mongo_unittest_third_group
 
   - name: unit_test_group4
@@ -1172,7 +1172,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_fourth_group,-intermediate_debug,-remote_exec_test
+            --test_tag_filters=mongo_unittest_fourth_group,-intermediate_debug,-final_target
             --build_tag_filters=mongo_unittest_fourth_group
 
   - name: unit_test_group5
@@ -1198,7 +1198,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_fifth_group,-intermediate_debug,-remote_exec_test
+            --test_tag_filters=mongo_unittest_fifth_group,-intermediate_debug,-final_target
             --build_tag_filters=mongo_unittest_fifth_group
 
   - name: unit_test_group6
@@ -1224,7 +1224,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_sixth_group,-intermediate_debug,-remote_exec_test
+            --test_tag_filters=mongo_unittest_sixth_group,-intermediate_debug,-final_target
             --build_tag_filters=mongo_unittest_sixth_group
 
   - name: unit_test_group7
@@ -1250,7 +1250,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_seventh_group,-intermediate_debug,-remote_exec_test
+            --test_tag_filters=mongo_unittest_seventh_group,-intermediate_debug,-final_target
             --build_tag_filters=mongo_unittest_seventh_group
 
   - name: unit_test_group8
@@ -1276,7 +1276,7 @@ tasks:
           bazel_args: >-
             --config=evg
             --include_autogenerated_targets=True
-            --test_tag_filters=mongo_unittest_eighth_group,-intermediate_debug,-remote_exec_test
+            --test_tag_filters=mongo_unittest_eighth_group,-intermediate_debug,-final_target
             --build_tag_filters=mongo_unittest_eighth_group
 
   - name: monitor_build_status