SERVER-100274: 8.1 Build and Verification Branching Task (#33136)

GitOrigin-RevId: 359cbe95216ffaf1a6173884f6519b3d408f1fb5
2025-03-06 15:59:40 -05:00 · 2025-03-06 15:59:40 -05:00 · e0770aed31
parent 589fd5cea1
commit e0770aed31
10 changed files with 86 additions and 100 deletions
--- a/buildscripts/generate_version_expansions.py
+++ b/buildscripts/generate_version_expansions.py
@ -61,8 +61,8 @@ def generate_version_expansions():
            raise ValueError("Unable to parse version from stdin and no version.json provided")

    if version_parts[0]:
-        expansions["suffix"] = "latest"
-        expansions["src_suffix"] = "latest"
+        expansions["suffix"] = "v8.1-latest"
+        expansions["src_suffix"] = "v8.1-latest"
        expansions["is_release"] = "false"
    else:
        expansions["suffix"] = version_line
--- a/buildscripts/validate_evg_project_config.py
+++ b/buildscripts/validate_evg_project_config.py
@ -2,7 +2,6 @@ import os.path
 import re
 import subprocess
 import sys
-from collections import defaultdict

 import structlog
 import typer
@ -16,7 +15,6 @@ DEFAULT_LOCAL_EVG_AUTH_CONFIG = os.path.expanduser("~/.evergreen.yml")

 DEFAULT_EVG_PROJECT_NAME = "mongodb-mongo-master"
 DEFAULT_EVG_NIGHTLY_PROJECT_NAME = "mongodb-mongo-master-nightly"
-DEFAULT_EVG_PROJECT_CONFIG = "etc/evergreen.yml"
 DEFAULT_EVG_NIGHTLY_PROJECT_CONFIG = "etc/evergreen_nightly.yml"

 ALLOWABLE_EVG_VALIDATE_MESSAGE_REGEXES = [
@ -41,76 +39,23 @@ def main(
    evg_project_config_map = {evg_project_name: DEFAULT_EVG_NIGHTLY_PROJECT_CONFIG}
    if evg_project_name == DEFAULT_EVG_PROJECT_NAME:
        evg_project_config_map = {
-            DEFAULT_EVG_PROJECT_NAME: DEFAULT_EVG_PROJECT_CONFIG,
            DEFAULT_EVG_NIGHTLY_PROJECT_NAME: DEFAULT_EVG_NIGHTLY_PROJECT_CONFIG,
        }

-    shared_evg_validate_messages = []
-    error_on_evg_validate_messages = defaultdict(list)
-
-    num_of_projects = len(evg_project_config_map)
    evergreen_bin = find_evergreen_binary("evergreen")
-    for project, project_config in evg_project_config_map.items():
+    for _, project_config in evg_project_config_map.items():
        cmd = [
            evergreen_bin,
            "--config",
            evg_auth_config,
-            "validate",
-            "--project",
-            project,
+            "evaluate",
            "--path",
            project_config,
        ]
        LOGGER.info(f"Running command: {cmd}")
-        result = subprocess.run(cmd, capture_output=True, text=True)
-        interesting_messages = result.stdout.strip().split("\n")[:-1]
+        subprocess.run(cmd, capture_output=True, text=True, check=True)

-        for message in interesting_messages:
-            if num_of_projects > 1 and any(
-                regex.match(message)
-                for regex in ALLOWABLE_IF_NOT_IN_ALL_PROJECTS_EVG_VALIDATE_MESSAGE_REGEXES
-            ):
-                shared_evg_validate_messages.append(message)
-                continue
-            if any(regex.match(message) for regex in ALLOWABLE_EVG_VALIDATE_MESSAGE_REGEXES):
-                continue
-            error_on_evg_validate_messages[project].append(message)
-
-    error_on_shared_evg_validate_messages = []
-    for message in set(shared_evg_validate_messages):
-        if shared_evg_validate_messages.count(message) == num_of_projects:
-            error_on_shared_evg_validate_messages.append(message)
-
-    exit_code = 0
-    all_configs = list(evg_project_config_map.values())
-    all_projects = list(evg_project_config_map.keys())
-
-    for project, errors in error_on_evg_validate_messages.items():
-        if len(errors) > 0:
-            exit_code = 1
-            project_config = evg_project_config_map[project]
-            LOGGER.info(HORIZONTAL_LINE)
-            LOGGER.error(f"Config '{project_config}' for '{project}' evergreen project has errors:")
-            for error in errors:
-                LOGGER.error(error)
-
-    if len(error_on_shared_evg_validate_messages) > 0:
-        exit_code = 1
-        LOGGER.info(HORIZONTAL_LINE)
-        LOGGER.error(
-            f"Configs {all_configs} for evergreen projects {all_projects} have errors"
-            f" (they can be fixed in either config):"
-        )
-        for error in error_on_shared_evg_validate_messages:
-            LOGGER.error(error)
-
-    if exit_code == 0:
-        LOGGER.info(HORIZONTAL_LINE)
-        LOGGER.info(
-            f"Config(s) {all_configs} for evergreen project(s) {all_projects} is(are) valid"
-        )
-
-    sys.exit(exit_code)
+    sys.exit(0)


 if __name__ == "__main__":
--- a/copy.bara.sky
+++ b/copy.bara.sky
@ -12,12 +12,12 @@ core.workflow(
    name = "default",
    origin = git.origin(
        url = sourceUrl,
-        ref = "master",
+        ref = "v8.1",
    ),
    destination = git.destination(
        url = destinationUrl,
-        fetch = "master",
-        push = "master",
+        fetch = "v8.1",
+        push = "v8.1",
    ),
    # Change path to the folder you want to publish publicly
    origin_files = glob(["**"], exclude = ["src/mongo/db/modules/**"]),
--- a/copy.bara.staging.sky
+++ b/copy.bara.staging.sky
@ -7,12 +7,12 @@ core.workflow(
    name = "default",
    origin = git.origin(
        url = sourceUrl,
-        ref = "master",
+        ref = "v8.1",
    ),
    destination = git.destination(
        url = destinationUrl,
-        fetch = "master",
-        push = "master",
+        fetch = "v8.1",
+        push = "v8.1",
    ),
    # Change path to the folder you want to publish publicly
    origin_files = glob(["**"], exclude = ["src/mongo/db/modules/**"]),
--- a/etc/evergreen_nightly.yml
+++ b/etc/evergreen_nightly.yml
@ -14,40 +14,40 @@ include:

  - filename: etc/evergreen_yml_components/variants/misc/task_generation.yml
  ### Uncomment when using this file for a LTS or Rapid release branch. ###
-  # - filename: etc/evergreen_yml_components/variants/misc/misc.yml
+  - filename: etc/evergreen_yml_components/variants/misc/misc.yml

  - filename: etc/evergreen_yml_components/variants/amazon/test_release.yml
  ### Uncomment when using this file for a LTS or Rapid release branch. ###
-  # - filename: etc/evergreen_yml_components/variants/amazon/test_dev.yml
+  - filename: etc/evergreen_yml_components/variants/amazon/test_dev.yml

  - filename: etc/evergreen_yml_components/variants/debian/test_release.yml

  ### Comment out when using this file for a Rapid release branch. ###
-  - filename: etc/evergreen_yml_components/variants/ibm/test_release_master_and_lts_branches_only.yml
+  # - filename: etc/evergreen_yml_components/variants/ibm/test_release_master_and_lts_branches_only.yml

  - filename: etc/evergreen_yml_components/variants/macos/test_release.yml

  - filename: etc/evergreen_yml_components/variants/rhel/test_release.yml
  ### Uncomment when using this file for a LTS or Rapid release branch. ###
-  # - filename: etc/evergreen_yml_components/variants/rhel/test_dev.yml
+  - filename: etc/evergreen_yml_components/variants/rhel/test_dev.yml
  ### Uncomment when using this file for a LTS release branch. ###
  # - filename: etc/evergreen_yml_components/variants/rhel/test_dev_master_and_lts_branches_only.yml

  ### Uncomment when using this file for a LTS or Rapid release branch. ###
-  # - filename: etc/evergreen_yml_components/variants/sanitizer/test_dev.yml
+  - filename: etc/evergreen_yml_components/variants/sanitizer/test_dev.yml

  ### Uncomment when using this file for a LTS or Rapid release branch. ###
-  # filename: etc/evergreen_yml_components/variants/mongot/test_dev.yml
+  - filename: etc/evergreen_yml_components/variants/mongot/test_dev.yml

  - filename: etc/evergreen_yml_components/variants/suse/test_release.yml

  - filename: etc/evergreen_yml_components/variants/ubuntu/test_release.yml
  ### Uncomment when using this file for a LTS release branch. ###
-  # - filename: etc/evergreen_yml_components/variants/ubuntu/test_dev_master_and_lts_branches_only.yml
+  - filename: etc/evergreen_yml_components/variants/ubuntu/test_dev_master_and_lts_branches_only.yml

  - filename: etc/evergreen_yml_components/variants/windows/test_release.yml
  ### Uncomment when using this file for a LTS or Rapid release branch. ###
-  # - filename: etc/evergreen_yml_components/variants/windows/test_dev.yml
+  - filename: etc/evergreen_yml_components/variants/windows/test_dev.yml
  ### Uncomment when using this file for a LTS release branch. ###
  # - filename: etc/evergreen_yml_components/variants/windows/test_dev_master_and_lts_branches_only.yml

@ -56,7 +56,31 @@ parameters:
    value: "etc/evergreen_nightly.yml"
    description: "path to this file"

+commit_queue_aliases:
+  - variant: "commit-queue"
+    task: "^(bazel_.*|run_.*|compile_.*|lint_.*|jsCore|version_gen_validation|validate_commit_message|resmoke_validation_tests|buildscripts_test)$"
+    variant_tags: []
+    task_tags: []
+  - variant: "^(amazon-linux2-arm64-static-compile|linux-x86-dynamic-compile-required)$"
+    variant_tags: []
+    task_tags: ["clang_tidy"]
+  - variant: "^(al2023-arm64-sep-benchmark)$"
+    variant_tags: []
+    task_tags: ["benchmarks_sep"]
+
 patch_aliases:
+  - alias: required
+    variant_tags: ["required"]
+    task: ".*"
+  - alias: query
+    variant: "^(.*query-patch-only|amazon-linux2-arm64-static-compile)$"
+    task: ".*"
+  - alias: query-quick
+    variant: "^(.*query-quick-patch-only|linux-x86-dynamic-compile)$"
+    task: ".*"
+  - alias: security
+    variant: "^(.*security-patch-only|linux-x86-dynamic-compile|windows-compile-required)$"
+    task: ".*"
  - alias: bazel
    description: "Runs bazel build system tests"
    variant_tags: ["bazel_check"]
@ -65,3 +89,25 @@ patch_aliases:
    description: "Runs bazel build system tests"
    variant_tags: ["bazel_check"]
    task: ".*"
+  - alias: search
+    description: "Runs all $search, $vectorSearch tests"
+    variant: ".*"
+    task: "^.*search*"
+  - alias: required-and-mongot-e2e-tests
+    description: >
+      This alias selects all tasks in all required variants and all tasks in variants that use a real mongot.
+      Once more e2e tests have been contributed and thus our coverage is better, this alias can be refined
+      to only select tests that use a real mongot.
+    variant_tags: ["required", "mongot_e2e"]
+    task: ".*"
+  - alias: codecoverage
+    description: "Run tests and report code coverage"
+    variant: ".*-coverage"
+    task: ".*"
+  - alias: unittestcoverage
+    description: "Run unit tests and report code coverage"
+    variant: ".*-coverage"
+    task: "^(compile_and_run_unittests_.*|bazel_coverage)$"
+  - alias: cluster_scalability
+    variant_tags: ["cluster_scalability_only"]
+    task_tags: ["cluster_scalability_only"]
--- a/etc/evergreen_yml_components/variants/amazon/test_dev.yml
+++ b/etc/evergreen_yml_components/variants/amazon/test_dev.yml
@ -143,8 +143,8 @@ buildvariants:

  - <<: *amazon_linux2_arm64_static_compile_variant_dependency
    name: enterprise-amazon-linux2-arm64
-    display_name: "* Amazon Linux 2 arm64 Atlas Enterprise"
-    tags: ["suggested", "forbid_tasks_tagged_with_experimental"]
+    display_name: "! Amazon Linux 2 arm64 Atlas Enterprise"
+    tags: ["required", "forbid_tasks_tagged_with_experimental"]
    cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
    run_on:
      - amazon2-arm64-atlas-latest-small
--- a/etc/evergreen_yml_components/variants/misc/misc.yml
+++ b/etc/evergreen_yml_components/variants/misc/misc.yml
@ -18,10 +18,7 @@ buildvariants:
      burn_in_tag_exclude_build_variants: >-
        macos-debug-suggested
        ubuntu2204-arm64-bazel-compile
-      burn_in_tag_include_build_variants: >-
-        enterprise-rhel-8-64-bit-inmem
-        enterprise-rhel-8-64-bit-multiversion
-        amazon-linux2-arm64-try-sbe-engine
+      burn_in_tag_include_build_variants:
      burn_in_tag_compile_task_dependency: archive_dist_test
      compile_variant: &amazon_linux2_arm64_static_compile_variant_name amazon-linux2-arm64-static-compile
    depends_on:
--- a/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml
+++ b/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml
@ -319,7 +319,7 @@ buildvariants:
          - rhel8.8-medium

  - <<: *linux_debug_aubsan_compile_variant_dependency
-    name: rhel8-debug-aubsan-lite-all-feature-flags-required
+    name: rhel8-debug-aubsan-lite-required
    display_name: "! Shared Library {A,UB}SAN Enterprise RHEL 8 DEBUG (all feature flags)"
    tags: ["required"]
    cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
@ -330,7 +330,6 @@ buildvariants:
      # To force disable feature flags even on the all feature flags variant, please use this file:
      # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml
      test_flags: >-
-        --runAllFeatureFlagTests
        --excludeWithAnyTags=incompatible_aubsan
    tasks:
      - name: jsCore
--- a/etc/evergreen_yml_components/variants/windows/test_dev.yml
+++ b/etc/evergreen_yml_components/variants/windows/test_dev.yml
@ -114,7 +114,7 @@ buildvariants:
      - name: run_bazel_TG

  - <<: *windows_compile_variant_dependency
-    name: enterprise-windows-all-feature-flags-required
+    name: enterprise-windows-required
    display_name: "! Enterprise Windows Server 2022 (all feature flags) essential tasks"
    tags: ["required", "forbid_tasks_tagged_with_experimental"]
    cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
@ -136,23 +136,22 @@ buildvariants:
      # To force disable feature flags even on the all feature flags variant, please use this file:
      # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml
      test_flags: >-
-        --runAllFeatureFlagTests
        --excludeWithAnyTags=incompatible_with_windows_tls
      external_auth_jobs_max: 1
    tasks:
-      - name: .development_critical !.requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.incompatible_all_feature_flags .requires_execution_on_windows_patch_build
-      - name: .development_critical .requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.incompatible_all_feature_flags .requires_execution_on_windows_patch_build
+      - name: .development_critical !.requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.requires_all_feature_flags .requires_execution_on_windows_patch_build
+      - name: .development_critical .requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.requires_all_feature_flags .requires_execution_on_windows_patch_build
        distros:
          - windows-2022-xlarge
-      - name: .release_critical !.requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.incompatible_all_feature_flags .requires_execution_on_windows_patch_build
-      - name: .release_critical .requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.incompatible_all_feature_flags .requires_execution_on_windows_patch_build
+      - name: .release_critical !.requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.requires_all_feature_flags .requires_execution_on_windows_patch_build
+      - name: .release_critical .requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.requires_all_feature_flags .requires_execution_on_windows_patch_build
        distros:
          - windows-2022-xlarge

  # This Windows build variant exists to run all tasks that have been excluded from running on the required Windows build variant.
  # See SERVER-79037 for how the essential set of tasks was computed.
  - <<: *windows_compile_variant_dependency
-    name: enterprise-windows-all-feature-flags-non-essential
+    name: enterprise-windows-non-essential
    display_name: "* Enterprise Windows Server 2022 (all feature flags) non-essential tasks"
    tags: ["suggested"]
    cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
@ -162,11 +161,11 @@ buildvariants:
      <<: *windows_essential_expansions
      burn_in_tests_build_variant: enterprise-windows-all-feature-flags-non-essential
    tasks:
-      - name: .development_critical !.requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.incompatible_all_feature_flags
-      - name: .development_critical .requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.incompatible_all_feature_flags
+      - name: .development_critical !.requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.requires_all_feature_flags
+      - name: .development_critical .requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.requires_all_feature_flags
        distros:
          - windows-2022-xlarge
-      - name: .release_critical !.requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.incompatible_all_feature_flags
-      - name: .release_critical .requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.incompatible_all_feature_flags
+      - name: .release_critical !.requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.requires_all_feature_flags
+      - name: .release_critical .requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.requires_all_feature_flags
        distros:
          - windows-2022-xlarge
--- a/etc/system_perf.yml
+++ b/etc/system_perf.yml
@ -13,14 +13,14 @@ include:
  #
  - filename: evergreen/system_perf/master/base.yml
    module: dsi
-  - filename: evergreen/system_perf/master/compiles.yml
-    module: dsi
-  - filename: evergreen/system_perf/master/variants.yml
-    module: dsi
-  - filename: evergreen/system_perf/master/master_variants.yml
-    module: dsi
  - filename: evergreen/system_perf/shared_tasks.yml
    module: dsi
+  - filename: evergreen/system_perf/8.1/compiles.yml
+    module: dsi
+  - filename: evergreen/system_perf/8.1/variants.yml
+    module: dsi
+  - filename: evergreen/system_perf/8.1/genny_tasks.yml
+    module: dsi
  - filename: evergreen/system_perf/master/genny_tasks.yml
    module: dsi
  ### lint_yaml trim end ###