SERVER-111177: add build task to run by default (#41619)

GitOrigin-RevId: 546ffe8f1c6a3996cff1e7b3cc65431d257289dd

buildscripts/evergreen_gen_streams_build_and_push_task.py

@@ -14,12 +14,26 @@ from buildscripts.resmokelib.utils import evergreen_conn
 from buildscripts.util.fileops import write_file
 from buildscripts.util.read_config import read_config_file
 
-# This file is for generating the task that builds and publishes the streams docker image.
+# This file is for generating the task that builds and pushes the streams docker image.
-# depends_on is only evaluated on task creation/validation, so all dependencies must exist prior to streams_build_and_publish.
+# depends_on is only evaluated on task creation/validation, so all dependencies must exist prior to streams_build_and_push.
 # Streams currently depends on multiple generated test suite tasks, which is why this task must also be generated.
 
 
-def make_task(compile_variant: str, additional_dependencies: set[str]) -> Task:
+def make_task(compile_variant: str, additional_dependencies: set[str], push: str) -> Task:
+    taskPrefix = "streams_build_"
+    scriptArgs = ["./src/evergreen/streams_image_build_and_push.sh"]
+    dependencies = {
+        TaskDependency("archive_dist_test", compile_variant),
+    }
+    # Only depend on tests if pushing image
+    if push == "true":
+        taskPrefix += "and_push_"
+        scriptArgs.append("--push")
+        dependencies.add(TaskDependency("aggregation", compile_variant))
+        dependencies.add(TaskDependency(".streams_release_test"))
+        for dep in additional_dependencies:
+            dependencies.add(TaskDependency(dep))
+
     commands = [
         BuiltInCommand("manifest.load", {}),
         FunctionCall("git get project and add git tag"),
@@ -39,23 +53,17 @@ def make_task(compile_variant: str, additional_dependencies: set[str]) -> Task:
             {
                 "add_expansions_to_env": True,
                 "binary": "bash",
-                "args": ["./src/evergreen/streams_image_push.sh"],
+                "args": scriptArgs,
             },
         ),
     ]
-    dependencies = {
+    return Task(f"{taskPrefix}{compile_variant}", commands, dependencies)
-        TaskDependency("archive_dist_test", compile_variant),
-        TaskDependency("aggregation", compile_variant),
-        TaskDependency(".streams_release_test"),
-    }
-    for dep in additional_dependencies:
-        dependencies.add(TaskDependency(dep))
-    return Task(f"streams_build_and_publish_{compile_variant}", commands, dependencies)
 
 
 def main(
     expansions_file: Annotated[str, typer.Argument()] = "expansions.yml",
-    output_file: Annotated[str, typer.Option("--output-file")] = "streams_build_and_publish.json",
+    output_file: Annotated[str, typer.Option("--output-file")] = "streams_build_and_push.json",
+    push: Annotated[str, typer.Option("--push")] = "false",
 ):
     evg_api = evergreen_conn.get_evergreen_api()
     expansions = read_config_file(expansions_file)
@@ -65,33 +73,34 @@ def main(
     evg_version = evg_api.version_by_id(version_id)
     variant = evg_version.build_by_variant(build_variant_name)
     task_deps = []
-    for task in variant.get_tasks():
+    if push == "true":
-        if task.display_name not in required_tasks:
+        for task in variant.get_tasks():
-            continue
+            if task.display_name not in required_tasks:
-        if task.execution_tasks:
+                continue
-            # is a display task
+            if task.execution_tasks:
-            for child_task_id in task.execution_tasks:
+                # is a display task
-                child_task = evg_api.task_by_id(child_task_id)
+                for child_task_id in task.execution_tasks:
-                task_deps.append(child_task.display_name)
+                    child_task = evg_api.task_by_id(child_task_id)
-        else:
+                    task_deps.append(child_task.display_name)
-            # is not a display task
+            else:
-            task_deps.append(task.display_name)
+                # is not a display task
+                task_deps.append(task.display_name)
 
-        required_tasks.remove(task.display_name)
+            required_tasks.remove(task.display_name)
 
-    print(task_deps)
+        print(task_deps)
-    if required_tasks:
+        if required_tasks:
-        print("The following required tasks were not found", required_tasks)
+            print("The following required tasks were not found", required_tasks)
-        raise RuntimeError("Could not find all required tasks")
+            raise RuntimeError("Could not find all required tasks")
 
     distro = expansions.get("distro_id")
     compile_variant_name = expansions.get("compile_variant")
-    current_task_name = expansions.get("task_name", "streams_build_and_publish_gen")
+    current_task_name = expansions.get("task_name", "streams_build_and_push_gen")
 
     build_variant = BuildVariant(name=build_variant_name)
     build_variant.display_task(
         current_task_name.replace("_gen", ""),
-        [make_task(compile_variant_name, additional_dependencies=task_deps)],
+        [make_task(compile_variant_name, additional_dependencies=task_deps, push=push)],
         distros=[distro],
     )
     shrub_project = ShrubProject.empty()

buildscripts/evergreen_gen_streams_publish_manifest_task.py

@@ -40,8 +40,8 @@ def make_task(compile_variant: str) -> Task:
         ),
     ]
     dependencies = {
-        TaskDependency(f"streams_build_and_publish_{compile_variant.replace('-arm64', '')}"),
+        TaskDependency(f"streams_build_and_push_{compile_variant.replace('-arm64', '')}"),
-        TaskDependency(f"streams_build_and_publish_{compile_variant}"),
+        TaskDependency(f"streams_build_and_push_{compile_variant}"),
     }
     return Task(f"streams_publish_manifest_{compile_variant}", commands, dependencies)
 

etc/evergreen_yml_components/definitions.yml

@@ -1099,7 +1099,7 @@ functions:
         files:
           - powercycle_tasks.json
 
-  "streams build and publish":
+  "streams build":
     - command: manifest.load
     - *git_get_shallow_project
     - *f_expansions_write
@@ -1115,25 +1115,62 @@ functions:
         binary: bash
         args:
           - "src/evergreen/run_python_script.sh"
-          - "buildscripts/evergreen_gen_streams_build_and_publish_task.py"
+          - "buildscripts/evergreen_gen_streams_build_and_push_task.py"
           - "../expansions.yml"
-          - "--output-file=../streams_build_and_publish.json"
+          - "--output-file=../streams_build_only.json"
     - command: s3.put
-      display_name: "upload streams_build_and_publish.json"
+      display_name: "upload streams_build_only.json"
       params:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
-        local_file: streams_build_and_publish.json
+        local_file: streams_build_only.json
-        remote_file: ${project}/${build_variant}/${revision}/streams_build_and_publish/${task_name}-${build_id}.json
+        remote_file: ${project}/${build_variant}/${revision}/streams_build_only/${task_name}-${build_id}.json
         bucket: mciuploads
         permissions: public-read
         content_type: application/gzip
         display_name: Generated Task Config - Execution ${execution}
     - command: generate.tasks
-      display_name: "generate.tasks streams_build_and_publish.json"
+      display_name: "generate.tasks streams_build_only.json"
       params:
         files:
-          - streams_build_and_publish.json
+          - streams_build_only.json
+
+  "streams build and push":
+    - command: manifest.load
+    - *git_get_shallow_project
+    - *f_expansions_write
+    - *restore_git_history_and_tags
+    - *add_git_tag
+    - *kill_processes
+    - *cleanup_environment
+    - *set_up_venv
+    - *upload_pip_requirements
+    - *configure_evergreen_api_credentials
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - "src/evergreen/run_python_script.sh"
+          - "buildscripts/evergreen_gen_streams_build_and_push_task.py"
+          - "../expansions.yml"
+          - "--output-file=../streams_build_and_push.json"
+          - "--push=true"
+    - command: s3.put
+      display_name: "upload streams_build_and_push.json"
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: streams_build_and_push.json
+        remote_file: ${project}/${build_variant}/${revision}/streams_build_and_push/${task_name}-${build_id}.json
+        bucket: mciuploads
+        permissions: public-read
+        content_type: application/gzip
+        display_name: Generated Task Config - Execution ${execution}
+    - command: generate.tasks
+      display_name: "generate.tasks streams_build_and_push.json"
+      params:
+        files:
+          - streams_build_and_push.json
 
   "streams publish manifest":
     - command: manifest.load

etc/evergreen_yml_components/tasks/compile_tasks_nightly.yml

@@ -764,9 +764,9 @@ tasks:
         # This is added because of EVG-18211.
         # Without this we are adding extra dependencies on evergreen and it is causing strain
         omit_generated_tasks: true
-      - name: streams_build_and_publish_gen
+      - name: streams_build_and_push_gen
         variant: enterprise-amazon2-streams
-      - name: streams_build_and_publish_gen
+      - name: streams_build_and_push_gen
         variant: enterprise-amazon2-streams-arm64
     commands:
       - func: "streams publish manifest"
@@ -779,9 +779,9 @@ tasks:
         # This is added because of EVG-18211.
         # Without this we are adding extra dependencies on evergreen and it is causing strain
         omit_generated_tasks: true
-      - name: streams_build_and_publish_gen
+      - name: streams_build_and_push_gen
         variant: enterprise-amazon2023-streams
-      - name: streams_build_and_publish_gen
+      - name: streams_build_and_push_gen
         variant: enterprise-amazon2023-streams-arm64
     commands:
       - func: "streams publish manifest"

etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml

@@ -370,7 +370,7 @@ tasks:
           fallback_num_sub_suites: 2
           resmoke_jobs_max: 1
 
-  - name: streams_build_and_publish_gen
+  - name: streams_build_only_gen
     tags: ["assigned_to_jira_team_streams", "auxiliary"]
     depends_on:
       - name: version_gen
@@ -378,11 +378,23 @@ tasks:
         # This is added because of EVG-18211.
         # Without this we are adding extra dependencies on evergreen and it is causing strain
         omit_generated_tasks: true
-      # Update with all gen tasks to depend on here and the display name in evergreen_gen_streams_build_and_publish_task.py
+    commands:
+      - func: "streams build"
+
+  - name: streams_build_and_push_gen
+    tags: ["assigned_to_jira_team_streams", "auxiliary"]
+    depends_on:
+      - name: version_gen
+        variant: generate-tasks-for-version
+        # This is added because of EVG-18211.
+        # Without this we are adding extra dependencies on evergreen and it is causing strain
+        omit_generated_tasks: true
+      # Update with all gen tasks to depend on here and the display name in evergreen_gen_streams_build_and_push_task.py
       - name: streams_gen
       - name: streams_kafka_gen
+      - name: streams_aspio_gen
     commands:
-      - func: "streams build and publish"
+      - func: "streams build and push"
 
   # Temporary task running the jscore suite as a bazel target. To be removed with SERVER-103537.
   - name: bazel_test_jscore

etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml

@@ -201,6 +201,7 @@ buildvariants:
       - name: .requires_extra_system_deps .default .requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_all_feature_flags !.suggested_excluding_required__for_devprod_mitigation_only
         distros:
           - amazon2023-arm64-latest-large
+      - name: streams_build_only_gen
 
   - name: amazon-linux2023-arm64-tests-lto-pgo-bolt
     display_name: "Amazon Linux 2023 arm64 Atlas Enterprise (all feature flags) LTO/PGO/BOLT"

etc/evergreen_yml_components/variants/amazon/test_release.yml

@@ -520,7 +520,7 @@ buildvariants:
       - name: streams_s3
       - name: streams_kinesis
       - name: streams_aspio_gen
-      - name: streams_build_and_publish_gen
+      - name: streams_build_and_push_gen
 
   - name: enterprise-amazon2023-streams-arm64
     display_name: "Amazon Linux 2023 enterprise build with streams arm64"
@@ -564,7 +564,7 @@ buildvariants:
       - name: streams_s3
       - name: aggregation
       - name: streams_aspio_gen
-      - name: streams_build_and_publish_gen
+      - name: streams_build_and_push_gen
       # Only needed once to generate the streams manifest for both x86 and arm
       - name: streams_publish_manifest_al2023_gen
 
@@ -609,7 +609,7 @@ buildvariants:
       - name: streams_s3
       - name: streams_kinesis
       - name: streams_aspio_gen
-      - name: streams_build_and_publish_gen
+      - name: streams_build_and_push_gen
 
   - name: enterprise-amazon2-streams-arm64
     display_name: "Amazon Linux 2 enterprise build with streams arm64"
@@ -653,7 +653,7 @@ buildvariants:
       - name: streams_s3
       - name: streams_kinesis
       - name: streams_aspio_gen
-      - name: streams_build_and_publish_gen
+      - name: streams_build_and_push_gen
       # Only needed once to generate the streams manifest for both x86 and arm
       - name: streams_publish_manifest_gen
 

evergreen/streams_image_build_and_push.sh

@@ -67,4 +67,6 @@ docker tag "$IMAGE" "$IMAGE:$GITSHA-$TAG_SUFFIX"
 
 docker images
 
-docker push "$IMAGE:$GITSHA-$TAG_SUFFIX"
+if [ "$1" == "--push" ]; then
+    docker push "$IMAGE:$GITSHA-$TAG_SUFFIX"
+fi