SERVER-105450 Disallow evergreen warnings for unmatched selectors that aren't tag-based (#36619)

GitOrigin-RevId: 92675e114e318124607dc8a3d0660dcb3ec9782d

buildscripts/tests/test_validate_evg_project_config.py

@@ -0,0 +1,68 @@
+"""Unit tests for validate_evg_project_config.py"""
+
+import unittest
+
+from buildscripts.validate_evg_project_config import messages_to_report
+
+
+class TestMessagesToReport(unittest.TestCase):
+    def test_allowed_if_not_in_all_projects(self):
+        messages = [
+            "NOTICE: task 'my_task' defined but not used by any variants; consider using or disabling"
+        ]
+        (_, shared_evg_validate_messages) = messages_to_report(messages, num_of_projects=2)
+        self.assertEqual(
+            shared_evg_validate_messages,
+            messages,
+            "The message should have been allowed in shared.",
+        )
+
+    def test_in_all_projects(self):
+        messages = [
+            "NOTICE: task 'my_task' defined but not used by any variants; consider using or disabling"
+        ]
+        (error_on_evg_validate_messages, _) = messages_to_report(messages, num_of_projects=1)
+        self.assertEqual(
+            error_on_evg_validate_messages,
+            messages,
+            "The message should have been reported as an error.",
+        )
+
+    def test_unmatched_tag_selector(self):
+        messages = [
+            "buildvariant 'foo' has unmatched criteria: '!.bar'",
+            "buildvariant 'foo' has unmatched selector: '.bar .baz'",
+        ]
+        (error_on_evg_validate_messages, _) = messages_to_report(messages, num_of_projects=1)
+        self.assertEqual(
+            error_on_evg_validate_messages,
+            [],
+            "The messages should have been allowed.",
+        )
+
+    def test_unmatched_selector(self):
+        messages = [
+            "buildvariant 'foo' has unmatched criteria: 'bar', 'baz'",
+            # "buildvariant 'foo' has unmatched selector: 'bar'",
+        ]
+        (error_on_evg_validate_messages, _) = messages_to_report(messages, num_of_projects=1)
+        self.assertEqual(
+            error_on_evg_validate_messages,
+            messages,
+            "The messages should have been reported as errors.",
+        )
+
+    def test_unmatched_selector_in_combo_message(self):
+        messages = [
+            "buildvariant 'foo' has unmatched criteria: '.bar .baz', 'bam'",
+            "buildvariant 'foo' has unmatched selector: 'bar', '!.baz'",
+        ]
+        (error_on_evg_validate_messages, _) = messages_to_report(messages, num_of_projects=1)
+        self.assertEqual(
+            error_on_evg_validate_messages,
+            messages,
+            "The messages should have been reported as errors.",
+        )
+
+
+# 'foo' optional comma + whitespace

buildscripts/validate_evg_project_config.py

@@ -8,6 +8,8 @@ import structlog
 import typer
 from typing_extensions import Annotated
 
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from buildscripts.ciconfig.evergreen import find_evergreen_binary
 
 LOGGER = structlog.get_logger(__name__)
@@ -19,9 +21,16 @@ DEFAULT_EVG_NIGHTLY_PROJECT_NAME = "mongodb-mongo-master-nightly"
 DEFAULT_EVG_PROJECT_CONFIG = "etc/evergreen.yml"
 DEFAULT_EVG_NIGHTLY_PROJECT_CONFIG = "etc/evergreen_nightly.yml"
 
-ALLOWABLE_EVG_VALIDATE_MESSAGE_REGEXES = [
+UNMATCHED_REGEXES = [
     re.compile(r".*buildvariant .+ has unmatched selector: .+"),
     re.compile(r".*buildvariant .+ has unmatched criteria: .+"),
+]
+ALLOWABLE_EVG_VALIDATE_MESSAGE_REGEXES = [
+    # These regex match any number of repeated criteria that look like '.tag1 !.tag2'
+    # unless they do not start with a dot or exclamation mark (meaning they are not
+    # tag-based selectors)
+    re.compile(r".*buildvariant .+ has unmatched selector: (('[!.][^']*?'),?\s?)+$"),
+    re.compile(r".*buildvariant .+ has unmatched criteria: (('[!.][^']*?'),?\s?)+$"),
     re.compile(
         r".*task 'select_multiversion_binaries' defined but not used by any variants; consider using or disabling.*"
     ),  # this task is added to variants only alongside multiversion generated tasks
@@ -33,6 +42,22 @@ ALLOWABLE_IF_NOT_IN_ALL_PROJECTS_EVG_VALIDATE_MESSAGE_REGEXES = [
 HORIZONTAL_LINE = "-" * 100
 
 
+def messages_to_report(messages, num_of_projects):
+    shared_evg_validate_messages = []
+    error_on_evg_validate_messages = []
+    for message in messages:
+        if any(regex.match(message) for regex in ALLOWABLE_EVG_VALIDATE_MESSAGE_REGEXES):
+            continue
+        if num_of_projects > 1 and any(
+            regex.match(message)
+            for regex in ALLOWABLE_IF_NOT_IN_ALL_PROJECTS_EVG_VALIDATE_MESSAGE_REGEXES
+        ):
+            shared_evg_validate_messages.append(message)
+            continue
+        error_on_evg_validate_messages.append(message)
+    return (error_on_evg_validate_messages, shared_evg_validate_messages)
+
+
 def main(
     evg_project_name: Annotated[
         str, typer.Option(help="Evergreen project name")
@@ -68,16 +93,10 @@ def main(
         result = subprocess.run(cmd, capture_output=True, text=True)
         interesting_messages = result.stdout.strip().split("\n")[:-1]
 
-        for message in interesting_messages:
-            if any(regex.match(message) for regex in ALLOWABLE_EVG_VALIDATE_MESSAGE_REGEXES):
-                continue
-            if num_of_projects > 1 and any(
-                regex.match(message)
-                for regex in ALLOWABLE_IF_NOT_IN_ALL_PROJECTS_EVG_VALIDATE_MESSAGE_REGEXES
-            ):
-                shared_evg_validate_messages.append(message)
-                continue
-            error_on_evg_validate_messages[project].append(message)
+        (error_on_evg_validate_messages[project], allowed_if_not_shared) = messages_to_report(
+            interesting_messages, num_of_projects
+        )
+        shared_evg_validate_messages.extend(allowed_if_not_shared)
 
     error_on_shared_evg_validate_messages = []
     for message in set(shared_evg_validate_messages):
@@ -96,6 +115,10 @@ def main(
             LOGGER.error(f"Config '{project_config}' for '{project}' evergreen project has errors:")
             for error in errors:
                 LOGGER.error(error)
+                if any(regex.match(error) for regex in UNMATCHED_REGEXES):
+                    LOGGER.info(
+                        "Unmatched selector/criteria are allowed if they are tagged based (using '!' or '.'), but not if they directly name a task/task group"
+                    )
 
     if len(error_on_shared_evg_validate_messages) > 0:
         exit_code = 1

etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml

@@ -301,8 +301,8 @@ buildvariants:
       - name: sharded_collections_jscore_passthrough_gen
       - name: noPassthrough_gen
       - name: noPassthroughWithMongod_gen
-      - name: initial_sync_fuzzer_sanity_patch
-      - name: rollback_fuzzer_sanity_patch
+      - name: initial_sync_fuzzer_sanity_patch_gen
+      - name: rollback_fuzzer_sanity_patch_gen
       - name: query_cbr_heuristic_jscore_passthrough
         cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
       - name: query_cbr_heuristic_aggregation_passthrough
@@ -368,8 +368,8 @@ buildvariants:
       - name: sharded_collections_jscore_passthrough_gen
       - name: noPassthrough_gen
       - name: noPassthroughWithMongod_gen
-      - name: initial_sync_fuzzer_sanity_patch
-      - name: rollback_fuzzer_sanity_patch
+      - name: initial_sync_fuzzer_sanity_patch_gen
+      - name: rollback_fuzzer_sanity_patch_gen
 
   - <<: *enterprise-amazon-linux2023-arm64-all-feature-flags-template
     name: enterprise-amazon-linux2023-arm64-all-feature-flags-required-query-patch-only

etc/evergreen_yml_components/variants/rhel/test_dev_master_and_lts_branches_only.yml

@@ -97,7 +97,6 @@ buildvariants:
       - name: replica_sets_gen
       - name: .replica_sets .common
       - name: .replica_sets .multi_oplog !.encrypt
-      - name: replica_sets_max_mirroring_large_txns_format_gen
       - name: replica_sets_multi_stmt_txn_jscore_passthrough_gen
       - name: replica_sets_multi_stmt_txn_stepdown_jscore_passthrough_gen
         distros:
@@ -146,7 +145,6 @@ buildvariants:
       - name: concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer_gen
       - name: concurrency_sharded_multi_stmt_txn_gen
       - name: concurrency_sharded_multi_stmt_txn_with_balancer_gen
-      - name: concurrency_sharded_multi_stmt_txn_with_stepdowns_gen
       - name: concurrency_sharded_with_stepdowns_gen
       - name: concurrency_sharded_stepdown_terminate_kill_primary_with_balancer_gen
       - name: concurrency_sharded_initial_sync_gen
@@ -171,8 +169,6 @@ buildvariants:
       - name: .sharding .txns
       - name: sharding_gen
       - name: sharding_auth_audit_gen
-      - name: sharding_max_mirroring_opportunistic_secondary_targeting_ese_gen
-      - name: sharding_max_mirroring_opportunistic_secondary_targeting_ese_gcm_gen
       - name: sharding_csrs_continuous_config_stepdown_gen
       - name: sharded_multi_stmt_txn_jscore_passthrough_gen
         distros:

etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml

@@ -109,7 +109,6 @@ buildvariants:
       - name: .replica_sets !.encrypt !.fcbis
       - name: replica_sets_reconfig_jscore_passthrough_gen
       - name: replica_sets_reconfig_jscore_stepdown_passthrough_gen
-      - name: replica_sets_max_mirroring_large_txns_format_gen
       - name: .retry
       - name: .read_only
       - name: session_jscore_passthrough