diff --git a/buildscripts/evergreen_gen_streams_build_and_publish_task.py b/buildscripts/evergreen_gen_streams_build_and_push_task.py similarity index 63% rename from buildscripts/evergreen_gen_streams_build_and_publish_task.py rename to buildscripts/evergreen_gen_streams_build_and_push_task.py index 0dbe576fa35..75eda5380c5 100644 --- a/buildscripts/evergreen_gen_streams_build_and_publish_task.py +++ b/buildscripts/evergreen_gen_streams_build_and_push_task.py @@ -14,12 +14,26 @@ from buildscripts.resmokelib.utils import evergreen_conn from buildscripts.util.fileops import write_file from buildscripts.util.read_config import read_config_file -# This file is for generating the task that builds and publishes the streams docker image. -# depends_on is only evaluated on task creation/validation, so all dependencies must exist prior to streams_build_and_publish. +# This file is for generating the task that builds and pushes the streams docker image. +# depends_on is only evaluated on task creation/validation, so all dependencies must exist prior to streams_build_and_push. # Streams currently depends on multiple generated test suite tasks, which is why this task must also be generated. -def make_task(compile_variant: str, additional_dependencies: set[str]) -> Task: +def make_task(compile_variant: str, additional_dependencies: set[str], push: str) -> Task: + taskPrefix = "streams_build_" + scriptArgs = ["./src/evergreen/streams_image_build_and_push.sh"] + dependencies = { + TaskDependency("archive_dist_test", compile_variant), + } + # Only depend on tests if pushing image + if push == "true": + taskPrefix += "and_push_" + scriptArgs.append("--push") + dependencies.add(TaskDependency("aggregation", compile_variant)) + dependencies.add(TaskDependency(".streams_release_test")) + for dep in additional_dependencies: + dependencies.add(TaskDependency(dep)) + commands = [ BuiltInCommand("manifest.load", {}), FunctionCall("git get project and add git tag"), @@ -39,23 +53,17 @@ def make_task(compile_variant: str, additional_dependencies: set[str]) -> Task: { "add_expansions_to_env": True, "binary": "bash", - "args": ["./src/evergreen/streams_image_push.sh"], + "args": scriptArgs, }, ), ] - dependencies = { - TaskDependency("archive_dist_test", compile_variant), - TaskDependency("aggregation", compile_variant), - TaskDependency(".streams_release_test"), - } - for dep in additional_dependencies: - dependencies.add(TaskDependency(dep)) - return Task(f"streams_build_and_publish_{compile_variant}", commands, dependencies) + return Task(f"{taskPrefix}{compile_variant}", commands, dependencies) def main( expansions_file: Annotated[str, typer.Argument()] = "expansions.yml", - output_file: Annotated[str, typer.Option("--output-file")] = "streams_build_and_publish.json", + output_file: Annotated[str, typer.Option("--output-file")] = "streams_build_and_push.json", + push: Annotated[str, typer.Option("--push")] = "false", ): evg_api = evergreen_conn.get_evergreen_api() expansions = read_config_file(expansions_file) @@ -65,33 +73,34 @@ def main( evg_version = evg_api.version_by_id(version_id) variant = evg_version.build_by_variant(build_variant_name) task_deps = [] - for task in variant.get_tasks(): - if task.display_name not in required_tasks: - continue - if task.execution_tasks: - # is a display task - for child_task_id in task.execution_tasks: - child_task = evg_api.task_by_id(child_task_id) - task_deps.append(child_task.display_name) - else: - # is not a display task - task_deps.append(task.display_name) + if push == "true": + for task in variant.get_tasks(): + if task.display_name not in required_tasks: + continue + if task.execution_tasks: + # is a display task + for child_task_id in task.execution_tasks: + child_task = evg_api.task_by_id(child_task_id) + task_deps.append(child_task.display_name) + else: + # is not a display task + task_deps.append(task.display_name) - required_tasks.remove(task.display_name) + required_tasks.remove(task.display_name) - print(task_deps) - if required_tasks: - print("The following required tasks were not found", required_tasks) - raise RuntimeError("Could not find all required tasks") + print(task_deps) + if required_tasks: + print("The following required tasks were not found", required_tasks) + raise RuntimeError("Could not find all required tasks") distro = expansions.get("distro_id") compile_variant_name = expansions.get("compile_variant") - current_task_name = expansions.get("task_name", "streams_build_and_publish_gen") + current_task_name = expansions.get("task_name", "streams_build_and_push_gen") build_variant = BuildVariant(name=build_variant_name) build_variant.display_task( current_task_name.replace("_gen", ""), - [make_task(compile_variant_name, additional_dependencies=task_deps)], + [make_task(compile_variant_name, additional_dependencies=task_deps, push=push)], distros=[distro], ) shrub_project = ShrubProject.empty() diff --git a/buildscripts/evergreen_gen_streams_publish_manifest_task.py b/buildscripts/evergreen_gen_streams_publish_manifest_task.py index e88eddff1b2..a6a3b02f9b8 100644 --- a/buildscripts/evergreen_gen_streams_publish_manifest_task.py +++ b/buildscripts/evergreen_gen_streams_publish_manifest_task.py @@ -40,8 +40,8 @@ def make_task(compile_variant: str) -> Task: ), ] dependencies = { - TaskDependency(f"streams_build_and_publish_{compile_variant.replace('-arm64', '')}"), - TaskDependency(f"streams_build_and_publish_{compile_variant}"), + TaskDependency(f"streams_build_and_push_{compile_variant.replace('-arm64', '')}"), + TaskDependency(f"streams_build_and_push_{compile_variant}"), } return Task(f"streams_publish_manifest_{compile_variant}", commands, dependencies) diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index f6e684355d7..ba44c9fb084 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -1099,7 +1099,7 @@ functions: files: - powercycle_tasks.json - "streams build and publish": + "streams build": - command: manifest.load - *git_get_shallow_project - *f_expansions_write @@ -1115,25 +1115,62 @@ functions: binary: bash args: - "src/evergreen/run_python_script.sh" - - "buildscripts/evergreen_gen_streams_build_and_publish_task.py" + - "buildscripts/evergreen_gen_streams_build_and_push_task.py" - "../expansions.yml" - - "--output-file=../streams_build_and_publish.json" + - "--output-file=../streams_build_only.json" - command: s3.put - display_name: "upload streams_build_and_publish.json" + display_name: "upload streams_build_only.json" params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: streams_build_and_publish.json - remote_file: ${project}/${build_variant}/${revision}/streams_build_and_publish/${task_name}-${build_id}.json + local_file: streams_build_only.json + remote_file: ${project}/${build_variant}/${revision}/streams_build_only/${task_name}-${build_id}.json bucket: mciuploads permissions: public-read content_type: application/gzip display_name: Generated Task Config - Execution ${execution} - command: generate.tasks - display_name: "generate.tasks streams_build_and_publish.json" + display_name: "generate.tasks streams_build_only.json" params: files: - - streams_build_and_publish.json + - streams_build_only.json + + "streams build and push": + - command: manifest.load + - *git_get_shallow_project + - *f_expansions_write + - *restore_git_history_and_tags + - *add_git_tag + - *kill_processes + - *cleanup_environment + - *set_up_venv + - *upload_pip_requirements + - *configure_evergreen_api_credentials + - command: subprocess.exec + params: + binary: bash + args: + - "src/evergreen/run_python_script.sh" + - "buildscripts/evergreen_gen_streams_build_and_push_task.py" + - "../expansions.yml" + - "--output-file=../streams_build_and_push.json" + - "--push=true" + - command: s3.put + display_name: "upload streams_build_and_push.json" + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: streams_build_and_push.json + remote_file: ${project}/${build_variant}/${revision}/streams_build_and_push/${task_name}-${build_id}.json + bucket: mciuploads + permissions: public-read + content_type: application/gzip + display_name: Generated Task Config - Execution ${execution} + - command: generate.tasks + display_name: "generate.tasks streams_build_and_push.json" + params: + files: + - streams_build_and_push.json "streams publish manifest": - command: manifest.load diff --git a/etc/evergreen_yml_components/tasks/compile_tasks_nightly.yml b/etc/evergreen_yml_components/tasks/compile_tasks_nightly.yml index c21188beed9..d61e005bb27 100644 --- a/etc/evergreen_yml_components/tasks/compile_tasks_nightly.yml +++ b/etc/evergreen_yml_components/tasks/compile_tasks_nightly.yml @@ -764,9 +764,9 @@ tasks: # This is added because of EVG-18211. # Without this we are adding extra dependencies on evergreen and it is causing strain omit_generated_tasks: true - - name: streams_build_and_publish_gen + - name: streams_build_and_push_gen variant: enterprise-amazon2-streams - - name: streams_build_and_publish_gen + - name: streams_build_and_push_gen variant: enterprise-amazon2-streams-arm64 commands: - func: "streams publish manifest" @@ -779,9 +779,9 @@ tasks: # This is added because of EVG-18211. # Without this we are adding extra dependencies on evergreen and it is causing strain omit_generated_tasks: true - - name: streams_build_and_publish_gen + - name: streams_build_and_push_gen variant: enterprise-amazon2023-streams - - name: streams_build_and_publish_gen + - name: streams_build_and_push_gen variant: enterprise-amazon2023-streams-arm64 commands: - func: "streams publish manifest" diff --git a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml index 630d430d79a..81967df21fe 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml @@ -370,7 +370,7 @@ tasks: fallback_num_sub_suites: 2 resmoke_jobs_max: 1 - - name: streams_build_and_publish_gen + - name: streams_build_only_gen tags: ["assigned_to_jira_team_streams", "auxiliary"] depends_on: - name: version_gen @@ -378,11 +378,23 @@ tasks: # This is added because of EVG-18211. # Without this we are adding extra dependencies on evergreen and it is causing strain omit_generated_tasks: true - # Update with all gen tasks to depend on here and the display name in evergreen_gen_streams_build_and_publish_task.py + commands: + - func: "streams build" + + - name: streams_build_and_push_gen + tags: ["assigned_to_jira_team_streams", "auxiliary"] + depends_on: + - name: version_gen + variant: generate-tasks-for-version + # This is added because of EVG-18211. + # Without this we are adding extra dependencies on evergreen and it is causing strain + omit_generated_tasks: true + # Update with all gen tasks to depend on here and the display name in evergreen_gen_streams_build_and_push_task.py - name: streams_gen - name: streams_kafka_gen + - name: streams_aspio_gen commands: - - func: "streams build and publish" + - func: "streams build and push" # Temporary task running the jscore suite as a bazel target. To be removed with SERVER-103537. - name: bazel_test_jscore diff --git a/etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml index 192acd66e98..f9be36d6148 100644 --- a/etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/amazon/test_dev_master_branch_only.yml @@ -201,6 +201,7 @@ buildvariants: - name: .requires_extra_system_deps .default .requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_all_feature_flags !.suggested_excluding_required__for_devprod_mitigation_only distros: - amazon2023-arm64-latest-large + - name: streams_build_only_gen - name: amazon-linux2023-arm64-tests-lto-pgo-bolt display_name: "Amazon Linux 2023 arm64 Atlas Enterprise (all feature flags) LTO/PGO/BOLT" diff --git a/etc/evergreen_yml_components/variants/amazon/test_release.yml b/etc/evergreen_yml_components/variants/amazon/test_release.yml index d4b6ba4e245..627dde26c14 100644 --- a/etc/evergreen_yml_components/variants/amazon/test_release.yml +++ b/etc/evergreen_yml_components/variants/amazon/test_release.yml @@ -520,7 +520,7 @@ buildvariants: - name: streams_s3 - name: streams_kinesis - name: streams_aspio_gen - - name: streams_build_and_publish_gen + - name: streams_build_and_push_gen - name: enterprise-amazon2023-streams-arm64 display_name: "Amazon Linux 2023 enterprise build with streams arm64" @@ -564,7 +564,7 @@ buildvariants: - name: streams_s3 - name: aggregation - name: streams_aspio_gen - - name: streams_build_and_publish_gen + - name: streams_build_and_push_gen # Only needed once to generate the streams manifest for both x86 and arm - name: streams_publish_manifest_al2023_gen @@ -609,7 +609,7 @@ buildvariants: - name: streams_s3 - name: streams_kinesis - name: streams_aspio_gen - - name: streams_build_and_publish_gen + - name: streams_build_and_push_gen - name: enterprise-amazon2-streams-arm64 display_name: "Amazon Linux 2 enterprise build with streams arm64" @@ -653,7 +653,7 @@ buildvariants: - name: streams_s3 - name: streams_kinesis - name: streams_aspio_gen - - name: streams_build_and_publish_gen + - name: streams_build_and_push_gen # Only needed once to generate the streams manifest for both x86 and arm - name: streams_publish_manifest_gen diff --git a/evergreen/streams_image_push.sh b/evergreen/streams_image_build_and_push.sh similarity index 95% rename from evergreen/streams_image_push.sh rename to evergreen/streams_image_build_and_push.sh index ed619497334..b31e33b5d42 100644 --- a/evergreen/streams_image_push.sh +++ b/evergreen/streams_image_build_and_push.sh @@ -67,4 +67,6 @@ docker tag "$IMAGE" "$IMAGE:$GITSHA-$TAG_SUFFIX" docker images -docker push "$IMAGE:$GITSHA-$TAG_SUFFIX" +if [ "$1" == "--push" ]; then + docker push "$IMAGE:$GITSHA-$TAG_SUFFIX" +fi