SERVER-103394 Use "bazel test" Interface in Evergreen (#34625)

GitOrigin-RevId: 2b7574e27327c3f520d318e74dff8feb7662ccc5

.bazelrc

@@ -355,6 +355,12 @@ common:windows --//bazel/config:linkstatic=True
 # The only Windows compiler we support is MSVC.
 common:windows --//bazel/config:compiler_type=msvc
 
+# Data dependencies don't work in bazel run or bazel test on windows without this
+startup --windows_enable_symlinks
+common:windows --enable_runfiles
+build:windows --action_env=MSYS=winsymlinks:nativestrict
+test:windows --action_env=MSYS=winsymlinks:nativestrict
+
 # Always use clang on macos
 common:macos --//bazel/config:compiler_type=clang
 

bazel/separate_debug.bzl

@@ -438,6 +438,7 @@ def macos_extraction(ctx, cc_toolchain, inputs):
         DefaultInfo(
             files = depset(outputs),
             executable = output_bin if ctx.attr.type == "program" else None,
+            runfiles = ctx.attr.binary_with_debug[DefaultInfo].data_runfiles,
         ),
         create_new_ccinfo_library(ctx, cc_toolchain, output_bin, unstripped_static_bin, ctx.attr.cc_shared_library),
     ]

bazel/toolchains/cc/mongo_linux/mongo_gdb.bzl

@@ -29,6 +29,7 @@ sh_binary(
     srcs = ["working_dir.sh"],
     data = ["%s/bin/gdb"],
     env = {"PYTHONPATH": "%s/lib/python3.10", "PYTHONHOME": "%s"},
+    visibility = ["//visibility:public"],
 )
 """ % (ctx.attr.version, pythonhome, pythonhome),
         )

bazel/wrapper_hook/engflow_check.py

@@ -10,9 +10,14 @@ sys.path.append(REPO_ROOT)
 from bazel.wrapper_hook.wrapper_debug import wrapper_debug
 
 
+def setup_auth_wrapper():
+    from buildscripts.bazel_rules_mongo.engflow_auth.engflow_auth import setup_auth
+
+    setup_auth(verbose=False)
+
+
 def engflow_auth(args):
     start = time.time()
-    from buildscripts.bazel_rules_mongo.engflow_auth.engflow_auth import setup_auth
 
     args_str = " ".join(args)
     if (
@@ -22,5 +27,5 @@ def engflow_auth(args):
         and "--config public-release" not in args_str
     ):
         if os.environ.get("CI") is None and platform.machine().lower() not in {"ppc64le", "s390x"}:
-            setup_auth(verbose=False)
+            setup_auth_wrapper()
     wrapper_debug(f"engflow auth time: {time.time() - start}")

buildscripts/BUILD.bazel

@@ -226,3 +226,20 @@ sh_binary(
     name = "mount_drives",
     srcs = ["mount_drives.sh"],
 )
+
+py_binary(
+    name = "gather_failed_unittests",
+    srcs = ["gather_failed_unittests.py"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//buildscripts/util",
+        dependency(
+            "typer",
+            group = "core",
+        ),
+        dependency(
+            "pyyaml",
+            group = "core",
+        ),
+    ],
+)

buildscripts/gather_failed_unittests.py

@@ -0,0 +1,119 @@
+import os
+import shutil
+import xml.etree.ElementTree as ET
+from glob import glob
+from pathlib import Path
+from typing import List
+
+import typer
+
+
+def _collect_failed_tests(testlog_dir: str) -> List[str]:
+    failed_tests = []
+    for test_xml in glob(f"{testlog_dir}/**/test.xml", recursive=True):
+        testsuite = ET.parse(test_xml).getroot().find("testsuite")
+        testcase = testsuite.find("testcase")
+        test_file = testcase.attrib["name"]
+
+        if testcase.find("error") is not None:
+            failed_tests += [test_file]
+    return failed_tests
+
+
+def _relink_binaries_with_symbols(failed_tests: List[str]):
+    # Enable this when/if we want to strip debug symbols during linking unit tests
+    # print("Relinking unit tests without stripping debug symbols...")
+    with open(".bazel_build_flags", "r", encoding="utf-8") as f:
+        bazel_build_flags = f.read().strip()
+
+    bazel_build_flags.replace("--config=strip-debug-during-link", "")
+
+    relink_command = [
+        arg for arg in ["bazel", "build", *bazel_build_flags.split(" "), *failed_tests] if arg
+    ]
+
+    # Enable this when/if we want to strip debug symbols during linking unit tests
+    # print(f"Running command: {' '.join(relink_command)}")
+    # subprocess.run(
+    #     relink_command,
+    #     check=True,
+    # )
+
+    repro_test_command = " ".join(["test" if arg == "build" else arg for arg in relink_command])
+    with open(".failed_unittest_repro.txt", "w", encoding="utf-8") as f:
+        f.write(repro_test_command)
+    print(f"Repro command written to .failed_unittest_repro.txt: {repro_test_command}")
+
+
+def _copy_bins_to_upload(failed_tests: List[str], upload_bin_dir: str, upload_lib_dir: str) -> bool:
+    success = True
+    bazel_bin_dir = Path("./bazel-bin")
+    for failed_test in failed_tests:
+        full_binary_path = bazel_bin_dir / failed_test
+        binary_name = failed_test.split(os.sep)[-1]
+        files_to_upload = []
+        for pattern in [
+            "*.core",
+            "*.mdmp",
+            f"{binary_name}.debug",
+            f"{binary_name}.pdb",
+            f"{binary_name}.exe",
+            f"{binary_name}",
+        ]:
+            files_to_upload.extend(bazel_bin_dir.rglob(pattern))
+
+        # core dumps may be in the root directory
+        files_to_upload.extend(Path(".").rglob("*.core"))
+        files_to_upload.extend(Path(".").rglob("*.mdmp"))
+
+        if not files_to_upload:
+            print(f"Cannot locate the files to upload for ({failed_test})")
+            success = False
+            continue
+
+        for binary_file in files_to_upload:
+            new_binary_file = upload_bin_dir / binary_file.name
+            if not os.path.exists(new_binary_file):
+                print(f"Copying {binary_file} to {new_binary_file}")
+                shutil.copy(binary_file, new_binary_file)
+
+        dsym_dir = full_binary_path.with_suffix(".dSYM")
+        if dsym_dir.is_dir():
+            print(f"Copying dsym {dsym_dir} to {upload_bin_dir}")
+            shutil.copytree(dsym_dir, upload_bin_dir / dsym_dir.name, dirs_exist_ok=True)
+
+    # Copy debug symbols for dynamic builds
+    lib_dir = Path("bazel-bin/install/lib")
+    if lib_dir.is_dir():
+        print(f"Copying debug symbols from {lib_dir} to {upload_lib_dir}")
+        shutil.copytree(lib_dir, upload_lib_dir, dirs_exist_ok=True)
+    return success
+
+
+def main(testlog_dir: str = "bazel-testlogs"):
+    """Gather unit test binaries and debug symbols of failed unit tests based off of bazel test logs."""
+
+    os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", "."))
+
+    upload_bin_dir = Path("dist-unittests/bin")
+    upload_lib_dir = Path("dist-unittests/lib")
+    upload_bin_dir.mkdir(parents=True, exist_ok=True)
+    upload_lib_dir.mkdir(parents=True, exist_ok=True)
+
+    failed_tests = _collect_failed_tests(testlog_dir)
+    if not failed_tests:
+        print("No failed tests found.")
+        exit(0)
+
+    print(f"Found {len(failed_tests)} failed tests. Gathering binaries and debug symbols.")
+    _relink_binaries_with_symbols(failed_tests)
+
+    print("Copying binaries and debug symbols to upload directories.")
+    if not _copy_bins_to_upload(failed_tests, upload_bin_dir, upload_lib_dir):
+        print("Fatal error occurred during processing.")
+        # TODO: add slack notification
+        exit(1)
+
+
+if __name__ == "__main__":
+    typer.run(main)

buildscripts/simple_report.py

@@ -60,6 +60,7 @@ def try_combine_reports(out: Report):
         with open("report.json") as fh:
             report = json.load(fh)
             out["results"] += report["results"]
+            if "failures" in report:
                 out["failures"] += report["failures"]
     except NameError:
         pass

etc/evergreen.yml

@@ -104,7 +104,7 @@ parameters:
 
 commit_queue_aliases:
   - variant: "commit-queue"
-    task: "^(bazel_.*|run_.*|compile_.*|lint_.*|jsCore|check_for_noexcept|version_gen_validation|validate_commit_message|resmoke_validation_tests|buildscripts_test)$"
+    task: "^(bazel_.*|run_.*|unit_test_group.*|compile_.*|lint_.*|jsCore|check_for_noexcept|version_gen_validation|validate_commit_message|resmoke_validation_tests|buildscripts_test)$"
     variant_tags: []
     task_tags: []
   - variant: "^(amazon-linux2023-arm64-static-compile|linux-x86-dynamic-compile-required)$"
@@ -116,7 +116,7 @@ commit_queue_aliases:
 
 github_pr_aliases:
   - variant: "commit-queue"
-    task: "^(bazel_.*|run_.*|compile_.*|lint_.*|jsCore|check_for_noexcept|version_gen_validation|validate_commit_message|resmoke_validation_tests|buildscripts_test)$"
+    task: "^(bazel_.*|run_.*|unit_test_group.*|compile_.*|lint_.*|jsCore|check_for_noexcept|version_gen_validation|validate_commit_message|resmoke_validation_tests|buildscripts_test)$"
     variant_tags: []
     task_tags: []
   - variant: "^(amazon-linux2023-arm64-static-compile|linux-x86-dynamic-compile-required)$"
@@ -167,7 +167,7 @@ patch_aliases:
   - alias: unittestcoverage
     description: "Run unit tests and report code coverage"
     variant: ".*-coverage"
-    task: "^(compile_and_run_unittests_.*|bazel_coverage)$"
+    task: "^(unit_test_group.*|bazel_coverage)$"
   - alias: cluster_scalability
     variant_tags: ["cluster_scalability_only"]
     task_tags: ["cluster_scalability_only"]

etc/evergreen_timeouts.yml

@@ -99,24 +99,32 @@ overrides:
       exec_timeout: 840 # 14 hours
     - task: ^compile_.*
       exec_timeout: 840 # 14 hours
+    - task: ^unit_test_group.*
+      exec_timeout: 840 # 14 hours
 
   enterprise-rhel-83-s390x-shared:
     - task: ^archive_.*
       exec_timeout: 840 # 14 hours
     - task: ^compile_.*
       exec_timeout: 840 # 14 hours
+    - task: ^unit_test_group.*
+      exec_timeout: 840 # 14 hours
 
   enterprise-rhel-9-s390x:
     - task: ^archive_.*
       exec_timeout: 840 # 14 hours
     - task: ^compile_.*
       exec_timeout: 840 # 14 hours
+    - task: ^unit_test_group.*
+      exec_timeout: 840 # 14 hours
 
   enterprise-rhel-9-s390x-shared:
     - task: ^archive_.*
       exec_timeout: 840 # 14 hours
     - task: ^compile_.*
       exec_timeout: 840 # 14 hours
+    - task: ^unit_test_group.*
+      exec_timeout: 840 # 14 hours
 
   rhel-8-arm64:
     - task: aggregation_blockprocessing_fuzzer

etc/evergreen_yml_components/definitions.yml

@@ -1423,6 +1423,19 @@ functions:
       args:
         - "src/evergreen/bazel_compile.sh"
 
+  "bazel test sh": &bazel_test_sh
+    command: subprocess.exec
+    display_name: "bazel test sh"
+    type: test
+    params:
+      binary: bash
+      env:
+        evergreen_remote_exec: ${evergreen_remote_exec|off}
+        author_email: ${author_email}
+      add_expansions_to_env: true
+      args:
+        - "src/evergreen/bazel_test.sh"
+
   "bazel compile (gcc)":
     - *get_version_expansions
     - *apply_version_expansions
@@ -1447,6 +1460,12 @@ functions:
     - *f_expansions_write
     - *bazel_compile_sh
 
+  "bazel test":
+    - *get_version_expansions
+    - *apply_version_expansions
+    - *f_expansions_write
+    - *bazel_test_sh
+
   "generate clang-tidy report sh": &generate_clang_tidy_report_sh
     command: subprocess.exec
     display_name: "generate clang-tidy report"
@@ -1467,6 +1486,9 @@ functions:
     type: test
     params:
       binary: bash
+      env:
+        evergreen_remote_exec: ${evergreen_remote_exec|off}
+        author_email: ${author_email}
       args:
         - "src/evergreen/bazel_run.sh"
 
@@ -1498,17 +1520,6 @@ functions:
     - *f_expansions_write
     - *bazel_coverage_sh
 
-  "bazel test sh": &bazel_test_sh
-    command: subprocess.exec
-    display_name: "bazel test sh"
-    type: test
-    params:
-      binary: bash
-      continue_on_err: true
-      add_expansions_to_env: true
-      args:
-        - "src/evergreen/bazel_test.sh"
-
   "run resmoke suite with bazel":
     - *f_expansions_write
     - *bazel_test_sh
@@ -2413,7 +2424,10 @@ functions:
         optional: true
 
   ### Process & archive failed unittest artifacts ###
-  "gather failed unittests": &gather_failed_unittests
+  # This only works for unit tests executed under resmoke,
+  # see buildscripts/gather_failed_unittests.py for gathering failed unit test artifacts
+  # when running under bazel test
+  "gather failed unittests resmoke": &gather_failed_unittests_resmoke
     command: subprocess.exec
     params:
       binary: bash
@@ -2441,11 +2455,25 @@ functions:
       display_name: Unit tests - Execution ${execution}
       optional: true
 
+  "upload failed unittest repro": &upload_failed_unittest_repro
+    command: s3.put
+    params:
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      local_file: src/.failed_unittest_repro.txt
+      remote_file: ${project}/${build_variant}/${revision}/unittests/${build_id}-${task_name}-${execution}/run_failed_unit_tests_command.txt
+      bucket: mciuploads
+      permissions: public-read
+      content_type: text/plain
+      display_name: Command To Run Failed Unit Tests on Workstation
+      optional: true
+
   "save failed unittests":
     - *f_expansions_write
-    - *gather_failed_unittests
+    - *gather_failed_unittests_resmoke
     - *tar_failed_unittests
     - *archive_failed_unittests
+    - *upload_failed_unittest_repro
 
   ### Process & archive artifacts from hung processes ###
   "run hang analyzer":

etc/evergreen_yml_components/tasks/compile_tasks.yml

@@ -43,6 +43,7 @@ variables:
       - func: "fetch pgo profile"
     teardown_task:
       - func: "f_expansions_write"
+      - func: "create bazel test report"
       - func: "attach report"
       - func: "attach artifacts"
       - func: "attach local resmoke invocation"
@@ -456,222 +457,6 @@ tasks:
             --config=evg
             --build_tag_filters=mongo_library
 
-  - name: compile_and_run_unittests_first_group
-    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
-    depends_on:
-      - name: version_expansions_gen
-        variant: generate-tasks-for-version
-      - name: compile_libraries_for_unittests
-    exec_timeout_secs: 32400 # 9 hours
-    commands:
-      - func: "bazel compile"
-        vars:
-          targets: install-mongo_unittest_first_group
-          compiling_for_test: true
-          task_compile_flags: ${unittest_compile_flags}
-          bazel_args: >-
-            --config=evg
-            --include_autogenerated_targets=True
-      - func: "f_expansions_write"
-      - func: "run diskstats"
-      - func: "f_expansions_write"
-      - func: "monitor process threads"
-      - func: "collect system resource info"
-      - func: "run tests"
-        vars:
-          suite: unittests_first_group
-          install_dir: bazel-bin/install/bin
-          exec_timeout_secs: 32400 # 9 hours
-
-  - name: compile_and_run_unittests_second_group
-    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
-    depends_on:
-      - name: version_expansions_gen
-        variant: generate-tasks-for-version
-      - name: compile_libraries_for_unittests
-    exec_timeout_secs: 32400 # 9 hours
-    commands:
-      - func: "bazel compile"
-        vars:
-          targets: install-mongo_unittest_second_group
-          compiling_for_test: true
-          task_compile_flags: ${unittest_compile_flags}
-          bazel_args: >-
-            --config=evg
-            --include_autogenerated_targets=True
-      - func: "f_expansions_write"
-      - func: "run diskstats"
-      - func: "f_expansions_write"
-      - func: "monitor process threads"
-      - func: "collect system resource info"
-      - func: "run tests"
-        vars:
-          suite: unittests_second_group
-          install_dir: bazel-bin/install/bin
-          exec_timeout_secs: 32400 # 9 hours
-
-  - name: compile_and_run_unittests_third_group
-    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
-    depends_on:
-      - name: version_expansions_gen
-        variant: generate-tasks-for-version
-      - name: compile_libraries_for_unittests
-    exec_timeout_secs: 32400 # 9 hours
-    commands:
-      - func: "bazel compile"
-        vars:
-          targets: install-mongo_unittest_third_group
-          compiling_for_test: true
-          task_compile_flags: ${unittest_compile_flags}
-          bazel_args: >-
-            --config=evg
-            --include_autogenerated_targets=True
-      - func: "f_expansions_write"
-      - func: "run diskstats"
-      - func: "f_expansions_write"
-      - func: "monitor process threads"
-      - func: "collect system resource info"
-      - func: "run tests"
-        vars:
-          suite: unittests_third_group
-          install_dir: bazel-bin/install/bin
-          exec_timeout_secs: 32400 # 9 hours
-
-  - name: compile_and_run_unittests_fourth_group
-    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
-    depends_on:
-      - name: version_expansions_gen
-        variant: generate-tasks-for-version
-      - name: compile_libraries_for_unittests
-    exec_timeout_secs: 32400 # 9 hours
-    commands:
-      - func: "bazel compile"
-        vars:
-          targets: install-mongo_unittest_fourth_group
-          compiling_for_test: true
-          task_compile_flags: ${unittest_compile_flags}
-          bazel_args: >-
-            --config=evg
-            --include_autogenerated_targets=True
-      - func: "f_expansions_write"
-      - func: "run diskstats"
-      - func: "f_expansions_write"
-      - func: "monitor process threads"
-      - func: "collect system resource info"
-      - func: "run tests"
-        vars:
-          suite: unittests_fourth_group
-          install_dir: bazel-bin/install/bin
-          exec_timeout_secs: 32400 # 9 hours
-
-  - name: compile_and_run_unittests_fifth_group
-    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
-    depends_on:
-      - name: version_expansions_gen
-        variant: generate-tasks-for-version
-      - name: compile_libraries_for_unittests
-    exec_timeout_secs: 32400 # 9 hours
-    commands:
-      - func: "bazel compile"
-        vars:
-          targets: install-mongo_unittest_fifth_group
-          compiling_for_test: true
-          task_compile_flags: ${unittest_compile_flags}
-          bazel_args: >-
-            --config=evg
-            --include_autogenerated_targets=True
-      - func: "f_expansions_write"
-      - func: "run diskstats"
-      - func: "f_expansions_write"
-      - func: "monitor process threads"
-      - func: "collect system resource info"
-      - func: "run tests"
-        vars:
-          suite: unittests_fifth_group
-          install_dir: bazel-bin/install/bin
-          exec_timeout_secs: 32400 # 9 hours
-
-  - name: compile_and_run_unittests_sixth_group
-    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
-    depends_on:
-      - name: version_expansions_gen
-        variant: generate-tasks-for-version
-      - name: compile_libraries_for_unittests
-    exec_timeout_secs: 32400 # 9 hours
-    commands:
-      - func: "bazel compile"
-        vars:
-          targets: install-mongo_unittest_sixth_group
-          compiling_for_test: true
-          task_compile_flags: ${unittest_compile_flags}
-          bazel_args: >-
-            --config=evg
-            --include_autogenerated_targets=True
-      - func: "f_expansions_write"
-      - func: "run diskstats"
-      - func: "f_expansions_write"
-      - func: "monitor process threads"
-      - func: "collect system resource info"
-      - func: "run tests"
-        vars:
-          suite: unittests_sixth_group
-          install_dir: bazel-bin/install/bin
-          exec_timeout_secs: 32400 # 9 hours
-
-  - name: compile_and_run_unittests_seventh_group
-    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
-    depends_on:
-      - name: version_expansions_gen
-        variant: generate-tasks-for-version
-      - name: compile_libraries_for_unittests
-    exec_timeout_secs: 32400 # 9 hours
-    commands:
-      - func: "bazel compile"
-        vars:
-          targets: install-mongo_unittest_seventh_group
-          compiling_for_test: true
-          task_compile_flags: ${unittest_compile_flags}
-          bazel_args: >-
-            --config=evg
-            --include_autogenerated_targets=True
-      - func: "f_expansions_write"
-      - func: "run diskstats"
-      - func: "f_expansions_write"
-      - func: "monitor process threads"
-      - func: "collect system resource info"
-      - func: "run tests"
-        vars:
-          suite: unittests_seventh_group
-          install_dir: bazel-bin/install/bin
-          exec_timeout_secs: 32400 # 9 hours
-
-  - name: compile_and_run_unittests_eighth_group
-    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
-    depends_on:
-      - name: version_expansions_gen
-        variant: generate-tasks-for-version
-      - name: compile_libraries_for_unittests
-    exec_timeout_secs: 32400 # 9 hours
-    commands:
-      - func: "bazel compile"
-        vars:
-          targets: install-mongo_unittest_eighth_group
-          compiling_for_test: true
-          task_compile_flags: ${unittest_compile_flags}
-          bazel_args: >-
-            --config=evg
-            --include_autogenerated_targets=True
-      - func: "f_expansions_write"
-      - func: "run diskstats"
-      - func: "f_expansions_write"
-      - func: "monitor process threads"
-      - func: "collect system resource info"
-      - func: "run tests"
-        vars:
-          suite: unittests_eighth_group
-          install_dir: bazel-bin/install/bin
-          exec_timeout_secs: 32400 # 9 hours
-
   ## run_unittests ##
   - name: run_unittests_future_git_tag_multiversion
     tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
@@ -1733,18 +1518,32 @@ task_groups:
       - compile_all_but_not_unittests
 
   - <<: *compile_task_group_template
-    name: compile_test_parallel_unittest_stream_TG
+    name: run_unit_tests_TG
     max_hosts: -1
     tasks:
       - compile_libraries_for_unittests
-      - compile_and_run_unittests_first_group
+      - unit_test_group1
-      - compile_and_run_unittests_second_group
+      - unit_test_group2
-      - compile_and_run_unittests_third_group
+      - unit_test_group3
-      - compile_and_run_unittests_fourth_group
+      - unit_test_group4
-      - compile_and_run_unittests_fifth_group
+      - unit_test_group5
-      - compile_and_run_unittests_sixth_group
+      - unit_test_group6
-      - compile_and_run_unittests_seventh_group
+      - unit_test_group7
-      - compile_and_run_unittests_eighth_group
+      - unit_test_group8
+
+  - <<: *compile_task_group_template
+    name: run_unit_tests_no_sandbox_TG
+    max_hosts: -1
+    tasks:
+      - compile_libraries_for_unittests
+      - unit_test_group1_no_sandbox
+      - unit_test_group2_no_sandbox
+      - unit_test_group3_no_sandbox
+      - unit_test_group4_no_sandbox
+      - unit_test_group5_no_sandbox
+      - unit_test_group6_no_sandbox
+      - unit_test_group7_no_sandbox
+      - unit_test_group8_no_sandbox
 
   - <<: *compile_task_group_template
     name: compile_unittest_libaries_TG

etc/evergreen_yml_components/tasks/compile_tasks_shared.yml

@@ -17,6 +17,7 @@ variables:
       - func: "fetch pgo profile"
     teardown_task:
       - func: "f_expansions_write"
+      - func: "create bazel test report"
       - func: "attach report"
       - func: "attach artifacts"
       - func: "attach local resmoke invocation"

etc/evergreen_yml_components/tasks/misc_tasks.yml

@@ -1107,13 +1107,493 @@ tasks:
       - func: "get engflow creds"
       - func: "run resmoke suite with bazel"
         vars:
-          target: //buildscripts/resmokeconfig:core
+          targets: //buildscripts/resmokeconfig:core
-          args: >-
+          bazel_args: >-
             --config=no-remote-exec
             --test_output=all
             --workspace_status_command="bazel/resmoke/volatile_status.sh"
             --//bazel/resmoke:in_evergreen
 
+  - name: unit_test_group1_no_sandbox
+    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    exec_timeout_secs: 32400 # 9 hours
+    commands:
+      - func: "bazel compile"
+        vars:
+          targets: install-mongo_unittest_first_group
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+      - func: "f_expansions_write"
+      - func: "run diskstats"
+      - func: "f_expansions_write"
+      - func: "monitor process threads"
+      - func: "collect system resource info"
+      - func: "run tests"
+        vars:
+          suite: unittests_first_group
+          install_dir: bazel-bin/install/bin
+          exec_timeout_secs: 32400 # 9 hours
+
+  - name: unit_test_group2_no_sandbox
+    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    exec_timeout_secs: 32400 # 9 hours
+    commands:
+      - func: "bazel compile"
+        vars:
+          targets: install-mongo_unittest_second_group
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+      - func: "f_expansions_write"
+      - func: "run diskstats"
+      - func: "f_expansions_write"
+      - func: "monitor process threads"
+      - func: "collect system resource info"
+      - func: "run tests"
+        vars:
+          suite: unittests_second_group
+          install_dir: bazel-bin/install/bin
+          exec_timeout_secs: 32400 # 9 hours
+
+  - name: unit_test_group3_no_sandbox
+    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    exec_timeout_secs: 32400 # 9 hours
+    commands:
+      - func: "bazel compile"
+        vars:
+          targets: install-mongo_unittest_third_group
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+      - func: "f_expansions_write"
+      - func: "run diskstats"
+      - func: "f_expansions_write"
+      - func: "monitor process threads"
+      - func: "collect system resource info"
+      - func: "run tests"
+        vars:
+          suite: unittests_third_group
+          install_dir: bazel-bin/install/bin
+          exec_timeout_secs: 32400 # 9 hours
+
+  - name: unit_test_group4_no_sandbox
+    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    exec_timeout_secs: 32400 # 9 hours
+    commands:
+      - func: "bazel compile"
+        vars:
+          targets: install-mongo_unittest_fourth_group
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+      - func: "f_expansions_write"
+      - func: "run diskstats"
+      - func: "f_expansions_write"
+      - func: "monitor process threads"
+      - func: "collect system resource info"
+      - func: "run tests"
+        vars:
+          suite: unittests_fourth_group
+          install_dir: bazel-bin/install/bin
+          exec_timeout_secs: 32400 # 9 hours
+
+  - name: unit_test_group5_no_sandbox
+    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    exec_timeout_secs: 32400 # 9 hours
+    commands:
+      - func: "bazel compile"
+        vars:
+          targets: install-mongo_unittest_fifth_group
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+      - func: "f_expansions_write"
+      - func: "run diskstats"
+      - func: "f_expansions_write"
+      - func: "monitor process threads"
+      - func: "collect system resource info"
+      - func: "run tests"
+        vars:
+          suite: unittests_fifth_group
+          install_dir: bazel-bin/install/bin
+          exec_timeout_secs: 32400 # 9 hours
+
+  - name: unit_test_group6_no_sandbox
+    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    exec_timeout_secs: 32400 # 9 hours
+    commands:
+      - func: "bazel compile"
+        vars:
+          targets: install-mongo_unittest_sixth_group
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+      - func: "f_expansions_write"
+      - func: "run diskstats"
+      - func: "f_expansions_write"
+      - func: "monitor process threads"
+      - func: "collect system resource info"
+      - func: "run tests"
+        vars:
+          suite: unittests_sixth_group
+          install_dir: bazel-bin/install/bin
+          exec_timeout_secs: 32400 # 9 hours
+
+  - name: unit_test_group7_no_sandbox
+    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    exec_timeout_secs: 32400 # 9 hours
+    commands:
+      - func: "bazel compile"
+        vars:
+          targets: install-mongo_unittest_seventh_group
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+      - func: "f_expansions_write"
+      - func: "run diskstats"
+      - func: "f_expansions_write"
+      - func: "monitor process threads"
+      - func: "collect system resource info"
+      - func: "run tests"
+        vars:
+          suite: unittests_seventh_group
+          install_dir: bazel-bin/install/bin
+          exec_timeout_secs: 32400 # 9 hours
+
+  - name: unit_test_group8_no_sandbox
+    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    exec_timeout_secs: 32400 # 9 hours
+    commands:
+      - func: "bazel compile"
+        vars:
+          targets: install-mongo_unittest_eighth_group
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+      - func: "f_expansions_write"
+      - func: "run diskstats"
+      - func: "f_expansions_write"
+      - func: "monitor process threads"
+      - func: "collect system resource info"
+      - func: "run tests"
+        vars:
+          suite: unittests_eighth_group
+          install_dir: bazel-bin/install/bin
+          exec_timeout_secs: 32400 # 9 hours
+
+  - name: unit_test_group1
+    tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    commands:
+      - command: timeout.update
+        params:
+          exec_timeout_secs: 21600 # 6 hours
+      - func: "f_expansions_write"
+      - command: manifest.load
+      - func: "git get project and add git tag"
+      - func: "f_expansions_write"
+      - func: "kill processes"
+      - func: "cleanup environment"
+      - func: "set up venv"
+      - func: "upload pip requirements"
+      - func: "get engflow creds"
+      - func: "enable bazel test report creation"
+      - func: "f_expansions_write"
+      - func: "bazel test"
+        vars:
+          targets: //src/mongo/...
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          test_timeout_sec: 600
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+            --test_tag_filters=mongo_unittest_first_group,-intermediate_target
+            --build_tag_filters=mongo_unittest_first_group
+
+  - name: unit_test_group2
+    tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    commands:
+      - command: timeout.update
+        params:
+          exec_timeout_secs: 21600 # 6 hours
+      - func: "f_expansions_write"
+      - command: manifest.load
+      - func: "git get project and add git tag"
+      - func: "f_expansions_write"
+      - func: "kill processes"
+      - func: "cleanup environment"
+      - func: "set up venv"
+      - func: "upload pip requirements"
+      - func: "get engflow creds"
+      - func: "enable bazel test report creation"
+      - func: "f_expansions_write"
+      - func: "bazel test"
+        vars:
+          targets: //src/mongo/...
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          test_timeout_sec: 600
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+            --test_tag_filters=mongo_unittest_second_group,-intermediate_target,-tracing_test
+            --build_tag_filters=mongo_unittest_second_group
+
+  - name: unit_test_group3
+    tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    commands:
+      - command: timeout.update
+        params:
+          exec_timeout_secs: 21600 # 6 hours
+      - func: "f_expansions_write"
+      - command: manifest.load
+      - func: "git get project and add git tag"
+      - func: "f_expansions_write"
+      - func: "kill processes"
+      - func: "cleanup environment"
+      - func: "set up venv"
+      - func: "upload pip requirements"
+      - func: "get engflow creds"
+      - func: "enable bazel test report creation"
+      - func: "f_expansions_write"
+      - func: "bazel test"
+        vars:
+          targets: //src/mongo/...
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          test_timeout_sec: 600
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+            --test_tag_filters=mongo_unittest_third_group,-intermediate_target
+            --build_tag_filters=mongo_unittest_third_group
+
+  - name: unit_test_group4
+    tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    commands:
+      - command: timeout.update
+        params:
+          exec_timeout_secs: 21600 # 6 hours
+      - func: "f_expansions_write"
+      - command: manifest.load
+      - func: "git get project and add git tag"
+      - func: "f_expansions_write"
+      - func: "kill processes"
+      - func: "cleanup environment"
+      - func: "set up venv"
+      - func: "upload pip requirements"
+      - func: "get engflow creds"
+      - func: "enable bazel test report creation"
+      - func: "f_expansions_write"
+      - func: "bazel test"
+        vars:
+          targets: //src/mongo/...
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          test_timeout_sec: 600
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+            --test_tag_filters=mongo_unittest_fourth_group,-intermediate_target
+            --build_tag_filters=mongo_unittest_fourth_group
+
+  - name: unit_test_group5
+    tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    commands:
+      - command: timeout.update
+        params:
+          exec_timeout_secs: 21600 # 6 hours
+      - func: "f_expansions_write"
+      - command: manifest.load
+      - func: "git get project and add git tag"
+      - func: "f_expansions_write"
+      - func: "kill processes"
+      - func: "cleanup environment"
+      - func: "set up venv"
+      - func: "upload pip requirements"
+      - func: "get engflow creds"
+      - func: "enable bazel test report creation"
+      - func: "f_expansions_write"
+      - func: "bazel test"
+        vars:
+          targets: //src/mongo/...
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          test_timeout_sec: 600
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+            --test_tag_filters=mongo_unittest_fifth_group,-intermediate_target
+            --build_tag_filters=mongo_unittest_fifth_group
+
+  - name: unit_test_group6
+    tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    commands:
+      - command: timeout.update
+        params:
+          exec_timeout_secs: 21600 # 6 hours
+      - func: "f_expansions_write"
+      - command: manifest.load
+      - func: "git get project and add git tag"
+      - func: "f_expansions_write"
+      - func: "kill processes"
+      - func: "cleanup environment"
+      - func: "set up venv"
+      - func: "upload pip requirements"
+      - func: "get engflow creds"
+      - func: "enable bazel test report creation"
+      - func: "f_expansions_write"
+      - func: "bazel test"
+        vars:
+          targets: //src/mongo/...
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          test_timeout_sec: 600
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+            --test_tag_filters=mongo_unittest_sixth_group,-intermediate_target
+            --build_tag_filters=mongo_unittest_sixth_group
+
+  - name: unit_test_group7
+    tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    commands:
+      - command: timeout.update
+        params:
+          exec_timeout_secs: 21600 # 6 hours
+      - func: "f_expansions_write"
+      - command: manifest.load
+      - func: "git get project and add git tag"
+      - func: "f_expansions_write"
+      - func: "kill processes"
+      - func: "cleanup environment"
+      - func: "set up venv"
+      - func: "upload pip requirements"
+      - func: "get engflow creds"
+      - func: "enable bazel test report creation"
+      - func: "f_expansions_write"
+      - func: "bazel test"
+        vars:
+          targets: //src/mongo/...
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          test_timeout_sec: 600
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+            --test_tag_filters=mongo_unittest_seventh_group,-intermediate_target
+            --build_tag_filters=mongo_unittest_seventh_group
+
+  - name: unit_test_group8
+    tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+      - name: compile_libraries_for_unittests
+    commands:
+      - command: timeout.update
+        params:
+          exec_timeout_secs: 21600 # 6 hours
+      - func: "f_expansions_write"
+      - command: manifest.load
+      - func: "git get project and add git tag"
+      - func: "f_expansions_write"
+      - func: "kill processes"
+      - func: "cleanup environment"
+      - func: "set up venv"
+      - func: "upload pip requirements"
+      - func: "get engflow creds"
+      - func: "enable bazel test report creation"
+      - func: "f_expansions_write"
+      - func: "bazel test"
+        vars:
+          targets: //src/mongo/...
+          compiling_for_test: true
+          task_compile_flags: ${unittest_compile_flags}
+          test_timeout_sec: 600
+          bazel_args: >-
+            --config=evg
+            --include_autogenerated_targets=True
+            --test_tag_filters=mongo_unittest_eighth_group,-intermediate_target
+            --build_tag_filters=mongo_unittest_eighth_group
+
   - name: monitor_build_status
     tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
     commands:

etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml

@@ -133,6 +133,7 @@ variables:
       - func: "get engflow creds"
     teardown_task:
       - func: "f_expansions_write"
+      - func: "create bazel test report"
       - func: "attach report"
       - func: "attach artifacts"
       - func: "attach local resmoke invocation"

etc/evergreen_yml_components/variants/amazon/test_dev.yml

@@ -95,7 +95,7 @@ buildvariants:
       evergreen_remote_exec: on
       skip_debug_link: true
     tasks:
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: compile_test_parallel_core_stream_and_pretty_printer_tests_TG
         distros:
           - amazon2023-arm64-latest-large-localstorage
@@ -186,7 +186,7 @@ buildvariants:
       - name: compile_all_but_not_unittests_TG
         distros:
           - amazon2023-arm64-xlarge-commitqueue
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - amazon2023-arm64-xlarge-commitqueue
       - name: compile_test_parallel_dbtest_stream_TG
@@ -374,6 +374,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - amazon2023-arm64-latest-large-m8g
+      - name: run_unit_tests_TG
+        distros:
+          - amazon2023-arm64-latest-large-m8g
       - name: .development_critical !.requires_large_host !.incompatible_community
       - name: .development_critical .requires_large_host !.incompatible_community
         distros:
@@ -406,6 +409,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - amazon2023-arm64-latest-large-m8g
+      - name: run_unit_tests_TG
+        distros:
+          - amazon2023-arm64-latest-large-m8g
       - name: .development_critical !.requires_large_host
       - name: .development_critical .requires_large_host
         distros:

etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml

@@ -86,7 +86,7 @@ buildvariants:
         --config=local
       compile_variant: *amazon-linux2023-arm64-local-compile
     tasks:
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: compile_test_parallel_core_stream_TG
 
   # Note that this task is currently optional;

etc/evergreen_yml_components/variants/amazon/test_release.yml

@@ -36,6 +36,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - amazon2-latest-large
+      - name: run_unit_tests_TG
+        distros:
+          - amazon2-latest-large
       - name: test_packages
         distros:
           - ubuntu2204-large
@@ -81,6 +84,10 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - amazon2-latest-large
+      # TODO(DEVPROD-17028): Re-enable when this distro has 750gb of space like the other larges
+      #- name: run_unit_tests_TG
+      #  distros:
+      #    - amazon2-latest-large
       - name: test_packages
         distros:
           - ubuntu2204-large
@@ -211,6 +218,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - amazon2023.3-large
+      - name: run_unit_tests_TG
+        distros:
+          - amazon2023.3-large
       - name: test_packages
         distros:
           - ubuntu2204-large
@@ -253,6 +263,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - amazon2023.3-large
+      - name: run_unit_tests_TG
+        distros:
+          - amazon2023.3-large
       - name: test_packages
         distros:
           - ubuntu2204-large
@@ -339,6 +352,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - amazon2023.3-arm64-large
+      - name: run_unit_tests_TG
+        distros:
+          - amazon2023.3-arm64-large
       - name: test_packages
         distros:
           - ubuntu2204-arm64-large
@@ -382,6 +398,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - amazon2023.3-arm64-large
+      - name: run_unit_tests_TG
+        distros:
+          - amazon2023.3-arm64-large
       - name: test_packages
         distros:
           - ubuntu2204-arm64-large
@@ -467,6 +486,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - amazon2023-latest-large
+      - name: run_unit_tests_TG
+        distros:
+          - amazon2023-latest-large
       - name: .development_critical !.requires_large_host
       - name: .development_critical .requires_large_host
         distros:
@@ -512,6 +534,10 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - amazon2-latest-large
+      # TODO(DEVPROD-17028): Re-enable when this distro has 750gb of space like the other larges
+      #- name: run_unit_tests_TG
+      #  distros:
+      #    - amazon2-latest-large
       - name: .development_critical !.requires_large_host
       - name: .development_critical .requires_large_host
         distros:
@@ -557,6 +583,10 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - amazon2-arm64-latest-large
+      # TODO(DEVPROD-17028): Re-enable when this distro has 750gb of space like the other larges
+      #- name: run_unit_tests_TG
+      #  distros:
+      #    - amazon2-arm64-latest-large
       - name: .development_critical !.requires_large_host
       - name: .development_critical .requires_large_host
         distros:

etc/evergreen_yml_components/variants/debian/test_release.yml

@@ -37,6 +37,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - debian12-large
+      - name: run_unit_tests_TG
+        distros:
+          - debian12-large
       - name: test_packages
         distros:
           - ubuntu2204-large
@@ -81,6 +84,10 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - debian12-large
+      # TODO(DEVPROD-17028): Re-enable when this distro has 750gb of space like the other larges
+      #- name: run_unit_tests_TG
+      #  distros:
+      #    - debian12-large
       - name: test_packages
         distros:
           - ubuntu2204-large

etc/evergreen_yml_components/variants/ibm/test_release_master_and_lts_branches_only.yml

@@ -67,7 +67,7 @@ buildvariants:
       - name: compile_test_serial_TG
         distros:
           - rhel81-power8-large
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - rhel81-power8-large
 
@@ -133,7 +133,7 @@ buildvariants:
       - name: compile_test_serial_TG
         distros:
           - rhel9-power-large
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - rhel9-power-large
 
@@ -197,7 +197,7 @@ buildvariants:
       - name: compile_test_serial_TG
         distros:
           - rhel83-zseries-large
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - rhel83-zseries-large
 
@@ -261,6 +261,6 @@ buildvariants:
       - name: compile_test_serial_TG
         distros:
           - rhel9-zseries-large
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - rhel9-zseries-large

etc/evergreen_yml_components/variants/macos/test_dev_master_branch_only.yml

@@ -17,7 +17,7 @@ buildvariants:
       resmoke_jobs_max: 6
     tasks:
       - name: compile_test_serial_TG
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: audit
       - name: auth_audit_gen
       - name: fle

etc/evergreen_yml_components/variants/macos/test_release.yml

@@ -26,7 +26,7 @@ buildvariants:
       compile_variant: macos-arm64
     tasks:
       - name: compile_test_and_package_serial_TG
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: .development_critical !.incompatible_community !.incompatible_mac
       - name: .release_critical !.incompatible_community !.incompatible_mac !publish_packages
 
@@ -50,7 +50,7 @@ buildvariants:
       compile_variant: enterprise-macos-arm64
     tasks:
       - name: compile_test_and_package_serial_TG
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: .development_critical !.incompatible_mac
       - name: .release_critical !.incompatible_mac !publish_packages
 
@@ -75,7 +75,7 @@ buildvariants:
       compile_variant: macos
     tasks:
       - name: compile_test_and_package_serial_TG
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: .development_critical !.incompatible_community !.incompatible_mac
       - name: .release_critical !.incompatible_community !.incompatible_mac !publish_packages
 
@@ -99,6 +99,6 @@ buildvariants:
       compile_variant: enterprise-macos
     tasks:
       - name: compile_test_and_package_serial_TG
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: .development_critical !.incompatible_mac
       - name: .release_critical !.incompatible_mac !publish_packages

etc/evergreen_yml_components/variants/rhel/test_dev.yml

@@ -95,7 +95,7 @@ buildvariants:
       build_mongot: true
       download_mongot_release: true
     tasks:
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: compile_test_parallel_core_stream_TG
       - name: compile_test_parallel_dbtest_stream_TG
       - name: compile_integration_and_test_parallel_stream_TG

etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml

@@ -35,7 +35,7 @@ variables:
     activate: true # These compile variants run on every commit to reduce latency of the auto-reverter.
     tasks:
       - name: compile_test_parallel_core_stream_TG
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: compile_test_parallel_dbtest_stream_TG
       - name: generate_buildid_to_debug_symbols_mapping
 
@@ -245,7 +245,7 @@ buildvariants:
         --mongodTlsCertificateKeyFile jstests/libs/server.pem
         --shellGRPC
     tasks:
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: compile_test_parallel_core_stream_TG
       - name: compile_test_parallel_dbtest_stream_TG
       - name: generate_buildid_to_debug_symbols_mapping
@@ -436,7 +436,7 @@ buildvariants:
       - name: compile_test_parallel_core_stream_TG
         distros:
           - rhel8.8-large
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - rhel8.8-large
       - name: compile_test_parallel_dbtest_stream_TG
@@ -467,7 +467,7 @@ buildvariants:
       - name: compile_test_parallel_core_stream_TG
         distros:
           - rhel8.8-large
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - rhel8.8-large
       - name: compile_test_parallel_dbtest_stream_TG
@@ -551,7 +551,7 @@ buildvariants:
       - name: compile_test_serial_TG
         distros:
           - rhel8.8-large
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - rhel8.8-large
         patch_only: true

etc/evergreen_yml_components/variants/rhel/test_release.yml

@@ -64,6 +64,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - rhel8.8-large
+      - name: run_unit_tests_TG
+        distros:
+          - rhel8.8-large
       - name: test_packages
         distros:
           - ubuntu2204-large
@@ -117,7 +120,7 @@ buildvariants:
       - name: compile_integration_and_test_parallel_stream_TG
         distros:
           - rhel8.8-large
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - rhel8.8-large
       - name: test_packages
@@ -163,6 +166,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - rhel8.8-arm64-large
+      - name: run_unit_tests_TG
+        distros:
+          - rhel8.8-arm64-large
       - name: test_packages
         distros:
           - ubuntu2204-arm64-small
@@ -205,7 +211,7 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - rhel8.8-arm64-large
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - rhel8.8-arm64-large
       - name: test_packages
@@ -252,6 +258,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - rhel93-large
+      - name: run_unit_tests_TG
+        distros:
+          - rhel93-large
       - name: test_packages
         distros:
           - ubuntu2204-large
@@ -295,7 +304,7 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - rhel93-large
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - rhel93-large
       - name: test_packages
@@ -341,7 +350,7 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - rhel93-arm64-large
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - rhel93-arm64-large
       - name: test_packages
@@ -386,6 +395,10 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - rhel93-arm64-large
+      # TODO(DEVPROD-17028): Re-enable when this distro has 750gb of space like the other larges
+      #- name: run_unit_tests_TG
+      #  distros:
+      #    - rhel93-arm64-large
       - name: test_packages
         distros:
           - ubuntu2204-arm64-small

etc/evergreen_yml_components/variants/sanitizer/test_dev.yml

@@ -16,7 +16,7 @@ variables:
     activate: true # These compile variants run on every commit to reduce latency of the auto-reverter.
     tasks:
       - name: compile_test_parallel_core_stream_TG
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: compile_test_parallel_dbtest_stream_TG
       - name: generate_buildid_to_debug_symbols_mapping
 
@@ -212,7 +212,7 @@ buildvariants:
         --mongodSetParameters="{internalQueryEnableAggressiveSpillsInGroup: true}"
     tasks:
       - name: compile_test_parallel_core_stream_TG
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: compile_test_parallel_dbtest_stream_TG
       - name: .clang_tidy
         distros:
@@ -246,7 +246,7 @@ buildvariants:
         --mongodSetParameters="{internalQueryEnableAggressiveSpillsInGroup: true}"
     tasks:
       - name: compile_test_parallel_core_stream_TG
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: compile_test_parallel_dbtest_stream_TG
       - name: compile_integration_and_test_parallel_stream_TG
       - name: compile_jstestshell_TG
@@ -385,7 +385,7 @@ buildvariants:
       compile_variant: rhel8-asan
     tasks:
       - name: compile_test_serial_TG
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: .aggfuzzer .common !.feature_flag_guarded !.incompatible_system_allocator
       - name: .jstestfuzz !.initsync !.feature_flag_guarded !.incompatible_system_allocator
       - name: generate_buildid_to_debug_symbols_mapping
@@ -436,7 +436,7 @@ buildvariants:
       <<: *enterprise-rhel8-debug-tsan-expansions-template
     tasks:
       - name: compile_test_parallel_core_stream_TG
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: compile_test_parallel_dbtest_stream_TG
       - name: compile_integration_and_test_parallel_stream_TG
       - name: compile_jstestshell_TG

etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml

@@ -198,7 +198,7 @@ buildvariants:
       - name: compile_test_parallel_core_stream_TG
         distros:
           - rhel8.8-xlarge
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - rhel8.8-xlarge
       - name: compile_test_parallel_dbtest_stream_TG
@@ -271,7 +271,7 @@ buildvariants:
       - name: compile_test_serial_TG
         distros:
           - windows-2022-xxlarge
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - windows-2022-xxlarge
       - name: compile_integration_and_test_no_audit_parallel_stream_TG
@@ -343,7 +343,7 @@ buildvariants:
         --developer_dir=/Applications/Xcode15.app
     tasks:
       - name: compile_test_serial_TG
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: .development_critical !.requires_large_host !.incompatible_development_variant !.incompatible_community !.incompatible_mac !.requires_all_feature_flags
       - name: .development_critical .requires_large_host !.incompatible_development_variant !.incompatible_community !.incompatible_mac !.requires_all_feature_flags
         distros:
@@ -371,7 +371,7 @@ buildvariants:
       resmoke_jobs_max: 6
     tasks:
       - name: compile_test_serial_TG
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: audit
       - name: auth_audit_gen
       - name: fle
@@ -566,7 +566,7 @@ buildvariants:
       - name: compile_test_parallel_core_stream_TG
         distros:
           - rhel8.8-large
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - rhel8.8-large
       - name: compile_test_parallel_dbtest_stream_TG
@@ -604,7 +604,7 @@ buildvariants:
         --mongodTlsCertificateKeyFile jstests/libs/server.pem
         --shellGRPC
     tasks:
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: compile_test_parallel_core_stream_TG
       - name: compile_test_parallel_dbtest_stream_TG
       - name: generate_buildid_to_debug_symbols_mapping

etc/evergreen_yml_components/variants/suse/test_release.yml

@@ -37,6 +37,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - suse15sp5-large
+      - name: run_unit_tests_TG
+        distros:
+          - suse15sp5-large
       - name: test_packages
         distros:
           - ubuntu2204-large
@@ -83,6 +86,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - suse15sp5-large
+      - name: run_unit_tests_TG
+        distros:
+          - suse15sp5-large
       - name: test_packages
         distros:
           - ubuntu2204-large

etc/evergreen_yml_components/variants/ubuntu/test_release.yml

@@ -38,6 +38,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - ubuntu2204-large
+      - name: run_unit_tests_TG
+        distros:
+          - ubuntu2204-large
       - name: test_packages
         distros:
           - ubuntu2204-large
@@ -93,6 +96,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - ubuntu2404-large
+      - name: run_unit_tests_TG
+        distros:
+          - ubuntu2204-large
       - name: test_packages
         distros:
           - ubuntu2404-large
@@ -147,6 +153,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - ubuntu2004-large
+      - name: run_unit_tests_TG
+        distros:
+          - ubuntu2004-large
       - name: test_packages
         distros:
           - ubuntu2204-large
@@ -193,6 +202,10 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - ubuntu2004-large
+      # TODO(DEVPROD-17028): Re-enable when this distro has 750gb of space like the other larges
+      #- name: run_unit_tests_TG
+      #  distros:
+      #    - ubuntu2004-large
       - name: test_packages
         distros:
           - ubuntu2204-large
@@ -252,7 +265,7 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - ubuntu2204-large
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - ubuntu2204-large
       - name: test_packages
@@ -345,6 +358,10 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - ubuntu2004-arm64-large
+      # TODO(DEVPROD-17028): Re-enable when this distro has 750gb of space like the other larges
+      #- name: run_unit_tests_TG
+      #  distros:
+      #    - ubuntu2004-arm64-large
       - name: test_packages
         distros:
           - ubuntu2204-arm64-small
@@ -391,6 +408,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - ubuntu2004-arm64-large
+      - name: run_unit_tests_TG
+        distros:
+          - ubuntu2004-arm64-large
       - name: test_packages
         distros:
           - ubuntu2204-arm64-small
@@ -434,7 +454,7 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - ubuntu2204-arm64-large
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - ubuntu2204-arm64-large
       - name: test_packages
@@ -483,6 +503,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - ubuntu2204-arm64-large
+      - name: run_unit_tests_TG
+        distros:
+          - ubuntu2204-arm64-large
       - name: test_packages
         distros:
           - ubuntu2204-arm64-large
@@ -528,6 +551,10 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - ubuntu2404-large
+      # TODO(DEVPROD-17028): Re-enable when this distro has 750gb of space like the other larges
+      #- name: run_unit_tests_TG
+      #  distros:
+      #    - ubuntu2404-large
       - name: test_packages
         distros:
           - ubuntu2404-large
@@ -577,6 +604,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - ubuntu2404-arm64-large
+      - name: run_unit_tests_TG
+        distros:
+          - ubuntu2404-arm64-large
       - name: test_packages
         distros:
           - ubuntu2404-arm64-large
@@ -618,6 +648,10 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - ubuntu2404-arm64-large
+      # TODO(DEVPROD-17028): Re-enable when this distro has 750gb of space like the other larges
+      #- name: run_unit_tests_TG
+      #  distros:
+      #    - ubuntu2404-arm64-large
       - name: test_packages
         distros:
           - ubuntu2404-arm64-large

etc/evergreen_yml_components/variants/windows/test_dev.yml

@@ -68,7 +68,7 @@ buildvariants:
       - name: compile_test_serial_no_unittests_TG
         distros:
           - windows-2022-xxxlarge-compile
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_no_sandbox_TG
         distros:
           - windows-2022-xxxlarge-compile
       - name: .development_critical .requires_compile_variant !.requires_large_host !.incompatible_development_variant !.incompatible_windows !.stitch .requires_execution_on_windows_patch_build

etc/evergreen_yml_components/variants/windows/test_release.yml

@@ -35,6 +35,9 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - windows-2022-xxlarge
+      - name: run_unit_tests_no_sandbox_TG
+        distros:
+          - windows-2022-xxlarge
       - name: .development_critical !.requires_large_host !.incompatible_community !.incompatible_windows
       - name: .development_critical .requires_large_host !.incompatible_community !.incompatible_windows
         distros:
@@ -80,7 +83,7 @@ buildvariants:
       - name: compile_test_and_package_serial_TG
         distros:
           - windows-2022-xxlarge
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_no_sandbox_TG
         distros:
           - windows-2022-xxlarge
       - name: .development_critical !.requires_large_host !.incompatible_windows

etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml

@@ -16,7 +16,7 @@ variables:
     activate: true # These compile variants run on every commit to reduce latency of the auto-reverter.
     tasks:
       - name: compile_test_parallel_core_stream_TG
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
       - name: compile_test_parallel_dbtest_stream_TG
       - name: generate_buildid_to_debug_symbols_mapping
 
@@ -149,7 +149,7 @@ buildvariants:
       - name: compile_test_serial_TG
         distros:
           - windows-2022-xxlarge
-      - name: compile_test_parallel_unittest_stream_TG
+      - name: run_unit_tests_TG
         distros:
           - windows-2022-xxlarge
       - name: burn_in_tests_gen

evergreen/bazel.sh

@@ -0,0 +1,7 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+
+source "$DIR/bazel_utility_functions.sh"
+(
+  cd $DIR/..
+  exec $(bazel_get_binary_path) "$@"
+)

evergreen/bazel_compile.sh

@@ -107,6 +107,7 @@ set -o pipefail
 eval echo "Execution environment: Targets: ${targets}"
 
 source ./evergreen/bazel_utility_functions.sh
+source ./evergreen/bazel_RBE_supported.sh
 
 if [[ "${evergreen_remote_exec}" != "on" ]]; then
   LOCAL_ARG="$LOCAL_ARG --jobs=auto"

evergreen/bazel_run.sh

@@ -20,6 +20,17 @@ activate_venv
 eval echo "Execution environment: Args: ${args} Target: ${target}"
 
 source ./evergreen/bazel_utility_functions.sh
+source ./evergreen/bazel_RBE_supported.sh
+
+if bazel_rbe_supported; then
+  LOCAL_ARG=""
+else
+  LOCAL_ARG="--config=local"
+fi
+
+if [[ "${evergreen_remote_exec}" != "on" ]]; then
+  LOCAL_ARG="--config=local"
+fi
 
 BAZEL_BINARY=$(bazel_get_binary_path)
 
@@ -37,14 +48,13 @@ fi
 
 # Print command being run to file that can be uploaded
 echo "python buildscripts/install_bazel.py" > bazel-invocation.txt
-echo "bazel run --verbose_failures $LOCAL_ARG ${args} ${target}" >> bazel-invocation.txt
+echo "bazel run --verbose_failures ${bazel_compile_flags} ${task_compile_flags} ${LOCAL_ARG} ${args} ${target}" >> bazel-invocation.txt
 
 # Run bazel command, retrying up to five times
 MAX_ATTEMPTS=5
 for ((i = 1; i <= $MAX_ATTEMPTS; i++)); do
-  eval $BAZEL_BINARY run --verbose_failures $LOCAL_ARG ${args} ${target} >> bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 1
+  eval $BAZEL_BINARY run --verbose_failures ${bazel_compile_flags} ${task_compile_flags} ${LOCAL_ARG} ${args} ${target}
-  if [ $i -lt $MAX_ATTEMPTS ]; then echo "Bazel failed to execute, retrying ($(($i + 1)) of $MAX_ATTEMPTS attempts)... " >> bazel_output.log 2>&1; fi
 done
 
-$python ./buildscripts/simple_report.py --test-name "bazel run ${args} ${target}" --log-file bazel_output.log --exit-code $RET
+# $python ./buildscripts/simple_report.py --test-name "bazel run ${args} ${target}" --log-file bazel_output.log --exit-code $RET
 exit $RET

evergreen/bazel_test.sh

@@ -2,8 +2,8 @@
 #   bazel_test [arguments]
 #
 # Required environment variables:
-# * ${target} - Test target
+# * ${targets} - Test targets
-# * ${args} - Extra command line args to pass to "bazel test"
+# * ${bazel_args} - Extra command line args to pass to "bazel test"
 
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
 . "$DIR/prelude.sh"
@@ -15,23 +15,65 @@ set -o verbose
 
 activate_venv
 
+set -o pipefail
+
 # Use `eval` to force evaluation of the environment variables in the echo statement:
-eval echo "Execution environment: Args: ${args} Target: ${target}"
+eval echo "Execution environment: Targets: ${targets}"
 
 source ./evergreen/bazel_utility_functions.sh
+source ./evergreen/bazel_RBE_supported.sh
+
+LOCAL_ARG=""
+if [[ "${evergreen_remote_exec}" != "on" ]]; then
+  LOCAL_ARG="$LOCAL_ARG --jobs=auto"
+fi
+
 BAZEL_BINARY=$(bazel_get_binary_path)
 
-# Print command being run to file that can be uploaded
+# Timeout is set here to avoid the build hanging indefinitely, still allowing
-echo "python buildscripts/install_bazel.py" > bazel-invocation.txt
+# for retries.
-echo "bazel test ${args} ${target}" >> bazel-invocation.txt
+TIMEOUT_CMD=""
+if [ -n "${build_timeout_seconds}" ]; then
+  TIMEOUT_CMD="timeout ${build_timeout_seconds}"
+fi
 
-set +e
+if is_ppc64le; then
+  LOCAL_ARG="$LOCAL_ARG --jobs=48"
+fi
 
-# use eval since some flags have quotes-in-quotes that are otherwise misinterpreted
+if is_s390x; then
-eval $BAZEL_BINARY test ${args} ${target}
+  LOCAL_ARG="$LOCAL_ARG --jobs=16"
-ret=$?
+fi
 
-set -e
+# If we are doing a patch build or we are building a non-push
+# build on the waterfall, then we don't need the --release
+# flag. Otherwise, this is potentially a build that "leaves
+# the building", so we do want that flag.
+if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then
+  echo "This is a non-release build."
+else
+  LOCAL_ARG="$LOCAL_ARG --config=public-release"
+fi
+
+if [ -n "${test_timeout_sec}" ]; then
+  bazel_args="${bazel_args} --test_timeout=${test_timeout_sec}"
+fi
+
+ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} --define=MONGO_VERSION=${version} ${patch_compile_flags}"
+echo ${ALL_FLAGS} > .bazel_build_flags
+
+set +o errexit
+
+for i in {1..3}; do
+  eval ${TIMEOUT_CMD} ${BAZEL_BINARY} test ${ALL_FLAGS} ${targets} 2>&1 | tee bazel_stdout.log \
+    && RET=0 && break || RET=$? && sleep 1
+  if [ $RET -eq 124 ]; then
+    echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..."
+  else
+    echo "Errors were found during the bazel test, failing the execution"
+    break
+  fi
+done
 
 # For a target //path:test, the undeclared test outputs are in
 # bazel-testlogs/path/test/test.outputs/outputs.zip
@@ -40,4 +82,14 @@ if [ -f $outputs ]; then
   unzip $outputs -d ../
 fi
 
-exit $ret
+set -o errexit
+
+if [[ $RET != 0 ]]; then
+  # The --config flag needs to stay consistent between invocations to avoid evicting the previous results.
+  # Strip out anything that isn't a --config flag that could interfere with the run command.
+  CONFIG_FLAGS=$(echo "${ALL_FLAGS}" | tr ' ' '\n' | grep -- '--config' | tr '\n' ' ')
+
+  eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:gather_failed_unittests
+fi
+
+exit $RET

evergreen/failed_unittests_gather.sh

@@ -8,7 +8,7 @@ set -eou pipefail
 # Only run on unit test tasks so we don't target mongod binaries from cores.
 if [ "${task_name}" != "run_dbtest" ] \
   && [[ ${task_name} != integration_tests* ]] \
-  && [[ "${task_name}" != *run_unittests* ]]; then
+  && [[ "${task_name}" != unit_test_group*_no_sandbox ]]; then
   echo "Not gathering failed unittests binaries as this is not a unittest task: ${task_name}"
   exit 0
 fi

evergreen/run_binary.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+$1

src/mongo/tracing/BUILD.bazel

@@ -55,7 +55,10 @@ mongo_cc_unit_test(
         "-Isrc/third_party/opentelemetry-cpp/exporters/otlp/include",
         "-Isrc/third_party/opentelemetry-cpp/sdk/include",
     ],
-    tags = ["mongo_unittest_second_group"],
+    tags = [
+        "mongo_unittest_second_group",
+        "tracing_test",
+    ],
     target_compatible_with = OTEL_TARGET_COMPATIBLE_WITH,
     deps = [
         ":tracing",

src/mongo/transport/grpc/BUILD.bazel

@@ -226,7 +226,9 @@ mongo_cc_unit_test(
         "//jstests/libs:test_crt_files",
         "//jstests/libs:test_pem_files",
     ],
-    tags = ["mongo_unittest_second_group"],
+    tags = [
+        "mongo_unittest_second_group",
+    ],
     target_compatible_with = GRPC_TARGET_COMPATIBLE_WITH,
     deps = [
         ":grpc_async_client_factory",