From b0245fd3b6b64cd190daa51a68f1abebf72a975f Mon Sep 17 00:00:00 2001 From: Zack Winter <3457246+zackwintermdb@users.noreply.github.com> Date: Thu, 25 Sep 2025 11:55:23 -0700 Subject: [PATCH] SERVER-109361 Update mciuploads settings (#41768) GitOrigin-RevId: 932dd24474cb3aa544e8e211073b40078d332ea5 --- etc/evergreen_yml_components/definitions.yml | 33 +++++++++- .../tasks/compile_tasks.yml | 45 +++++++++----- .../tasks/compile_tasks_nightly.yml | 60 ++++++++++++------- .../tasks/compile_tasks_shared.yml | 39 ++++++++---- .../tasks/misc_tasks.yml | 12 ++++ .../clusters_and_integrations/tasks.yml | 15 +++-- .../tasks.yml | 6 +- .../resmoke/server_divisions/query/tasks.yml | 3 +- .../variants/amazon/test_dev.yml | 2 + etc/expansions.default.yml | 2 + evergreen/prelude_mongo_task_generator.sh | 2 +- 11 files changed, 159 insertions(+), 60 deletions(-) diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 4dd7bfeb541..f6e684355d7 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -317,6 +317,27 @@ functions: local_file: src/mongo-binaries.zst optional: true + "fetch binary tgz shas": &fetch_tgz_binary_shas + command: s3.get + display_name: "fetch binary tgz shas" + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + remote_file: ${mongo_binaries}.sha256 + bucket: mciuploads + local_file: src/mongo-binaries.tgz.sha256 + + "fetch binary zstd shas": &fetch_zstd_binary_shas + command: s3.get + display_name: "fetch binary zstd shas" + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + remote_file: ${mongo_binaries}.zst.sha256 + bucket: mciuploads + local_file: src/mongo-binaries.zst.sha256 + optional: true + "fetch and verify binaries sha": &fetch_and_verify_binaries_sha command: subprocess.exec params: @@ -604,7 +625,8 @@ functions: local_file: src/bazel-bin/dist-test-debug.${ext|tgz} remote_file: ${mongo_debugsymbols} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/gzip} display_name: mongo-debugsymbols.${ext|tgz} @@ -838,6 +860,9 @@ functions: - *cleanup_environment - *fetch_venv - *adjust_venv + - *fetch_binaries + - *fetch_binaries_zstd + - *fetch_tgz_binary_shas - *fetch_and_verify_binaries_sha - *fetch_and_verify_binaries_sha_zstd - *fetch_jstestshell @@ -2852,7 +2877,8 @@ functions: local_file: mongo-unittests.tgz remote_file: ${project}/${build_variant}/${revision}/unittests/mongo-unittests-${build_id}-${task_name}-${execution}.tgz bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/gzip display_name: Unit tests - Execution ${execution} optional: true @@ -2934,7 +2960,8 @@ functions: local_file: src/mongo-hanganalyzer.tgz remote_file: ${project}/${build_variant}/${revision}/hanganalyzer/mongo-hanganalyzer-${build_id}-${task_name}-${execution}.tgz bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/gzip display_name: Hang Analyzer Output - Execution ${execution} optional: true diff --git a/etc/evergreen_yml_components/tasks/compile_tasks.yml b/etc/evergreen_yml_components/tasks/compile_tasks.yml index 01406463a47..e74802bd6e3 100644 --- a/etc/evergreen_yml_components/tasks/compile_tasks.yml +++ b/etc/evergreen_yml_components/tasks/compile_tasks.yml @@ -375,7 +375,8 @@ tasks: local_file: src/bazel-bin/mongo_benchmark-stripped.${ext|tgz} remote_file: ${mongo_benchmarks} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: Benchmarks @@ -402,7 +403,8 @@ tasks: local_file: src/bazel-bin/mongo_benchmark-debug.${ext|tgz} remote_file: ${mongo_benchmarks_debugsymbols} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: Benchmarks Debug @@ -577,7 +579,8 @@ tasks: local_file: src/bazel-bin/dbtest.${ext|tgz} remote_file: ${project}/${build_variant}/${revision}/dbtest/dbtest-${build_id}-${task_name}-${execution}.tgz bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: dbtest binary - Execution ${execution} optional: true @@ -638,7 +641,8 @@ tasks: local_file: src/bazel-bin/mongo-stripped.${ext|tgz} remote_file: ${mongo_jstestshell} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/gzip} display_name: Jstestshell - command: s3.put @@ -648,7 +652,8 @@ tasks: local_file: src/bazel-bin/mongo-stripped.${ext|tgz}.sha256 remote_file: ${mongo_jstestshell}.sha256 bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: text/plain display_name: Jstestshell SHA256 @@ -669,7 +674,8 @@ tasks: local_file: src/bazel-bin/mongo-debug.${ext|tgz} remote_file: ${mongo_jstestshell_debugsymbols} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/gzip} display_name: Jstestshell Debugsymbols @@ -703,7 +709,8 @@ tasks: local_file: "src/bazel-bin/stitch_support.${ext|tgz}" remote_file: "${project}/stitch-support/${build_variant}/${revision}/stitch-support-${version}.tgz" bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: "Stitch Support Library" @@ -758,7 +765,8 @@ tasks: local_file: "src/bazel-bin/mongo_crypt-stripped.${ext|tgz}" remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shared_v1_dev-${version}.${ext|tgz}" bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/tar} display_name: "Mongo Crypt Library dev" - command: s3.put @@ -768,7 +776,8 @@ tasks: local_file: "src/bazel-bin/mongo_crypt-debug.${ext|tgz}" remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shared_v1_debug-${version}.${ext|tgz}" bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/tar} display_name: "Mongo Crypt Library debug" @@ -791,7 +800,8 @@ tasks: local_file: "src/bazel-bin/mongo_crypt_shlib_test-stripped.${ext|tgz}" remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shlib_test-${version}.${ext|tgz}" bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/tar} display_name: "Mongo Crypt Shared Library Test" @@ -1073,7 +1083,8 @@ tasks: local_file: src/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz} aws_key: ${aws_key} bucket: ${push_bucket} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/gzip} remote_file: ${push_path}/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz} skip_existing: ${is_release} @@ -1084,7 +1095,8 @@ tasks: local_file: src/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig aws_key: ${aws_key} bucket: ${push_bucket} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/gzip} remote_file: ${push_path}/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig skip_existing: ${is_release} @@ -1094,7 +1106,8 @@ tasks: aws_secret: ${aws_secret} local_file: src/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1 aws_key: ${aws_key} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} bucket: ${push_bucket} content_type: text/plain remote_file: ${push_path}/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1 @@ -1104,7 +1117,8 @@ tasks: params: aws_secret: ${aws_secret} local_file: src/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256 - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} aws_key: ${aws_key} bucket: ${push_bucket} content_type: text/plain @@ -1117,7 +1131,8 @@ tasks: local_file: src/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5 aws_key: ${aws_key} bucket: ${push_bucket} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: text/plain remote_file: ${push_path}/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5 skip_existing: ${is_release} diff --git a/etc/evergreen_yml_components/tasks/compile_tasks_nightly.yml b/etc/evergreen_yml_components/tasks/compile_tasks_nightly.yml index 3f3313f2f70..c21188beed9 100644 --- a/etc/evergreen_yml_components/tasks/compile_tasks_nightly.yml +++ b/etc/evergreen_yml_components/tasks/compile_tasks_nightly.yml @@ -98,7 +98,8 @@ tasks: local_file: msi-files-signed.tgz remote_file: ${project}/${build_variant}/${revision}/artifacts/${build_id}-msi-files-signed.tgz bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: MSI Files @@ -195,7 +196,8 @@ tasks: local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} aws_key: ${aws_key} bucket: ${push_bucket} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/gzip} remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} skip_existing: ${is_release} @@ -206,7 +208,8 @@ tasks: local_file: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz} aws_key: ${aws_key} bucket: ${push_bucket} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/gzip} remote_file: ${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz} skip_existing: ${is_release} @@ -218,7 +221,8 @@ tasks: params: aws_secret: ${aws_secret} aws_key: ${aws_key} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} bucket: ${push_bucket} content_type: ${content_type|application/gzip} @@ -233,7 +237,8 @@ tasks: local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig aws_key: ${aws_key} bucket: ${push_bucket} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/gzip} remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig skip_existing: ${is_release} @@ -245,7 +250,8 @@ tasks: local_file: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig aws_key: ${aws_key} bucket: ${push_bucket} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/gzip} remote_file: ${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig skip_existing: ${is_release} @@ -256,7 +262,8 @@ tasks: params: aws_secret: ${aws_secret} aws_key: ${aws_key} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sig bucket: ${push_bucket} content_type: ${content_type|application/gzip} @@ -269,7 +276,8 @@ tasks: params: aws_key: ${aws_key} aws_secret: ${aws_secret} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} build_variants: ["enterprise-windows", "windows"] local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi bucket: ${push_bucket} @@ -283,7 +291,8 @@ tasks: aws_secret: ${aws_secret} local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1 aws_key: ${aws_key} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} bucket: ${push_bucket} content_type: text/plain remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1 @@ -295,7 +304,8 @@ tasks: aws_secret: ${aws_secret} local_file: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1 aws_key: ${aws_key} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} bucket: ${push_bucket} content_type: text/plain remote_file: ${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1 @@ -307,7 +317,8 @@ tasks: params: aws_secret: ${aws_secret} aws_key: ${aws_key} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha1 bucket: ${push_bucket} content_type: text/plain @@ -320,7 +331,8 @@ tasks: params: aws_key: ${aws_key} aws_secret: ${aws_secret} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} build_variants: ["enterprise-windows", "windows"] local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.sha1 bucket: ${push_bucket} @@ -333,7 +345,8 @@ tasks: params: aws_secret: ${aws_secret} local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256 - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} aws_key: ${aws_key} bucket: ${push_bucket} content_type: text/plain @@ -345,7 +358,8 @@ tasks: params: aws_secret: ${aws_secret} local_file: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256 - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} aws_key: ${aws_key} bucket: ${push_bucket} content_type: text/plain @@ -360,7 +374,8 @@ tasks: local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha256 aws_key: ${aws_key} bucket: ${push_bucket} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: text/plain remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha256 skip_existing: ${is_release} @@ -374,7 +389,8 @@ tasks: build_variants: ["enterprise-windows", "windows"] local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.sha256 bucket: ${push_bucket} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha256 skip_existing: ${is_release} content_type: text/plain @@ -386,7 +402,8 @@ tasks: local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5 aws_key: ${aws_key} bucket: ${push_bucket} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: text/plain remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5 skip_existing: ${is_release} @@ -398,7 +415,8 @@ tasks: local_file: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5 aws_key: ${aws_key} bucket: ${push_bucket} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: text/plain remote_file: ${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5 skip_existing: ${is_release} @@ -412,7 +430,8 @@ tasks: local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.md5 bucket: ${push_bucket} content_type: text/plain - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.md5 skip_existing: ${is_release} optional: true @@ -425,7 +444,8 @@ tasks: build_variants: ["enterprise-windows", "windows"] local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.md5 bucket: ${push_bucket} - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: text/plain remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.md5 skip_existing: ${is_release} diff --git a/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml b/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml index d79210b9b5c..6b7dc6c1180 100644 --- a/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml +++ b/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml @@ -153,7 +153,8 @@ tasks: local_file: src/bazel-bin/dist-test-stripped.${ext|tgz} remote_file: ${mongo_binaries} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/gzip # Sys-perf relies on this display name, please reach out before changing it. display_name: Binaries @@ -165,7 +166,8 @@ tasks: local_file: src/bazel-bin/dist-test-stripped.${ext|tgz}.sha256 remote_file: ${mongo_binaries}.sha256 bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: text/plain display_name: Binaries SHA256 - command: subprocess.exec @@ -345,7 +347,8 @@ tasks: local_file: src/version_info_mongod.txt remote_file: ${project}/${build_variant}/${version_id}/version-info-mongod-${task_id}.txt bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: text/plain display_name: mongod --version output @@ -399,7 +402,8 @@ tasks: local_file: "src/bazel-bin/mongo_crypt-stripped.${ext|tgz}" remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shared_v1-${version}.${ext|tgz}" bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/tar} display_name: "Mongo crypt Library" @@ -445,7 +449,8 @@ tasks: local_file: packages.tgz remote_file: ${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: Packages @@ -488,7 +493,8 @@ tasks: local_file: src/bazel-bin/dist-stripped.${ext|tgz} remote_file: ${project}/${build_variant}/${revision}/dist/mongo-${build_id}.${ext|tgz} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: Dist Tarball - command: s3.put @@ -499,7 +505,8 @@ tasks: local_file: src/bazel-bin/dist-debug.${ext|tgz} remote_file: ${project}/${build_variant}/${revision}/dist/mongo-${build_id}-debugsymbols.${ext|tgz} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: Dist Debugsymbols - command: subprocess.exec @@ -517,7 +524,8 @@ tasks: local_file: src/bazel-bin/mongo-stripped.${ext|tgz} remote_file: ${mongo_jstestshell} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/gzip} display_name: Jstestshell - command: s3.put @@ -527,7 +535,8 @@ tasks: local_file: src/bazel-bin/mongo-stripped.${ext|tgz}.sha256 remote_file: ${mongo_jstestshell}.sha256 bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: text/plain display_name: Jstestshell SHA256 - command: s3.put @@ -537,7 +546,8 @@ tasks: local_file: src/bazel-bin/mongo-debug.${ext|tgz} remote_file: ${mongo_jstestshell_debugsymbols} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/gzip} display_name: Jstestshell Debugsymbols - command: s3.put @@ -548,7 +558,8 @@ tasks: local_file: src/bazel-bin/mongocryptd-stripped.${ext|tgz} remote_file: ${mongo_cryptd} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/gzip} display_name: CryptD Binaries - command: s3.put @@ -559,7 +570,8 @@ tasks: local_file: src/bazel-bin/mongocryptd-debug.${ext|tgz} remote_file: ${mongo_cryptd_debugsymbols} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: ${content_type|application/gzip} display_name: CryptD Debugsymbols - func: "run packager.py" @@ -578,7 +590,8 @@ tasks: local_file: packages.tgz remote_file: ${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: Packages # We perform package task generally in the middle of larger task groups running diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index 37ee8dbfcb8..db73e2bdb84 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -1563,6 +1563,9 @@ tasks: - func: "set up venv" - func: "fetch packages" - func: "fetch binaries" + - func: "fetch binaries zstd" + - func: "fetch binary zstd shas" + - func: "fetch binary tgz shas" - func: "extract binaries" - func: "run selinux tests" vars: @@ -1581,6 +1584,9 @@ tasks: - func: "set up venv" - func: "fetch packages" - func: "fetch binaries" + - func: "fetch binaries zstd" + - func: "fetch binary zstd shas" + - func: "fetch binary tgz shas" - func: "extract binaries" - func: "run selinux tests" vars: @@ -1599,6 +1605,9 @@ tasks: - func: "set up venv" - func: "fetch packages" - func: "fetch binaries" + - func: "fetch binaries zstd" + - func: "fetch binary zstd shas" + - func: "fetch binary tgz shas" - func: "extract binaries" - func: "run selinux tests" vars: @@ -1617,6 +1626,9 @@ tasks: - func: "set up venv" - func: "fetch packages" - func: "fetch binaries" + - func: "fetch binaries zstd" + - func: "fetch binary zstd shas" + - func: "fetch binary tgz shas" - func: "extract binaries" - func: "run selinux tests" vars: diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml index 8203b89fa23..3059860568f 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml @@ -251,7 +251,8 @@ tasks: local_file: src/bazel-bin/first_half_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/first_half_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: Benchmarks @@ -287,7 +288,8 @@ tasks: local_file: src/bazel-bin/second_half_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/second_half_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: Benchmarks @@ -323,7 +325,8 @@ tasks: local_file: src/bazel-bin/sep_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/sep_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: Benchmarks @@ -1212,7 +1215,8 @@ tasks: local_file: src/bazel-bin/sharding_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/sharding_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: Benchmarks @@ -1247,7 +1251,8 @@ tasks: local_file: src/bazel-bin/bsoncolumn_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/bsoncolumn_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: Benchmarks diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml index d7e103921b6..cedfab46c98 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml @@ -139,7 +139,8 @@ tasks: local_file: src/bazel-bin/repl_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/repl_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: Benchmarks @@ -1829,7 +1830,8 @@ tasks: local_file: src/bazel-bin/storage_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/storage_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: Benchmarks diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml index 598385f300e..bfc482011db 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml @@ -2436,7 +2436,8 @@ tasks: local_file: src/bazel-bin/query_bm.${ext|tgz} remote_file: ${project}/${compile_variant}/${version_id}/binaries/query_bm-${revision_order_id}.${ext|tgz} bucket: mciuploads - permissions: public-read + permissions: ${mciuploads_binary_permissions|public-read} + visibility: ${mciuploads_binary_visibility|public} content_type: application/tar display_name: Benchmarks diff --git a/etc/evergreen_yml_components/variants/amazon/test_dev.yml b/etc/evergreen_yml_components/variants/amazon/test_dev.yml index db07de12e90..472a7bc8395 100644 --- a/etc/evergreen_yml_components/variants/amazon/test_dev.yml +++ b/etc/evergreen_yml_components/variants/amazon/test_dev.yml @@ -95,6 +95,8 @@ buildvariants: compile_variant: *amazon-linux2023-arm64-static-compile evergreen_remote_exec: on skip_debug_link: true + mciuploads_binary_permissions: private + mciuploads_binary_visibility: signed remote_link: true # Boost the priority on remote execution since this is the longest running task chain in # the required patch build. This priority is still lower than the priority on developer diff --git a/etc/expansions.default.yml b/etc/expansions.default.yml index 32b6ae8151b..7480216f788 100644 --- a/etc/expansions.default.yml +++ b/etc/expansions.default.yml @@ -27,6 +27,8 @@ npm_command: jstestfuzz tar: tar cmake_path: /opt/cmake/bin/cmake content_type: application/x-gzip +mciuploads_binary_permissions: public-read +mciuploads_binary_visibility: public jstestfuzz_concurrent_num_files: "10" curator_release: "latest" ext: tgz diff --git a/evergreen/prelude_mongo_task_generator.sh b/evergreen/prelude_mongo_task_generator.sh index a0bab0f4672..4c84508fbfa 100644 --- a/evergreen/prelude_mongo_task_generator.sh +++ b/evergreen/prelude_mongo_task_generator.sh @@ -1,6 +1,6 @@ function setup_mongo_task_generator { if [ ! -f mongo-task-generator ]; then - curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v3.5.0/mongo-task-generator --output mongo-task-generator + curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v3.5.1/mongo-task-generator --output mongo-task-generator chmod +x mongo-task-generator fi }