mirror of https://github.com/mongodb/mongo
1627 lines
59 KiB
YAML
1627 lines
59 KiB
YAML
# This file contains compile and related tasks
|
|
|
|
################################################
|
|
# Variable #
|
|
################################################
|
|
variables:
|
|
- &compile_bazel_task_group_template
|
|
name: compile_bazel_task_group_template
|
|
max_hosts: -1
|
|
tasks: []
|
|
setup_task:
|
|
- func: "override task timeout"
|
|
setup_group_can_fail_task: true
|
|
setup_group:
|
|
- command: manifest.load
|
|
- func: "git get project and add git tag"
|
|
- func: "set task expansion macros"
|
|
- func: "f_expansions_write"
|
|
- func: "set up venv"
|
|
- func: "upload pip requirements"
|
|
- func: "configure evergreen api credentials"
|
|
- func: "get buildnumber"
|
|
- func: "f_expansions_write"
|
|
- func: "set up credentials"
|
|
- func: "f_expansions_write"
|
|
- func: "get engflow creds"
|
|
|
|
# THIS HAS COPIES IN
|
|
# - etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml
|
|
# - etc/evergreen_yml_components/tasks/compile_tasks.yml
|
|
# - etc/evergreen_yml_components/tasks/compile_tasks_shared.yml
|
|
# ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES
|
|
- &compile_task_group_template
|
|
name: compile_task_group_template
|
|
max_hosts: 1
|
|
tasks: []
|
|
setup_task:
|
|
- func: "f_expansions_write"
|
|
- func: "set task expansion macros"
|
|
- func: "f_expansions_write"
|
|
- func: "override task timeout"
|
|
- func: "get engflow creds"
|
|
- func: "fetch pgo profile"
|
|
teardown_task:
|
|
- func: "f_expansions_write"
|
|
- func: "create bazel test report"
|
|
- func: "attach report"
|
|
- func: "attach artifacts"
|
|
- func: "attach local resmoke invocation"
|
|
- func: "attach multiversion download links"
|
|
- func: "kill processes"
|
|
- func: "save code coverage data"
|
|
- func: "save mongo coredumps"
|
|
- func: "generate hang analyzer tasks"
|
|
- func: "save failed unittests"
|
|
- func: "save bazel headers"
|
|
- func: "save bazel jvm dump"
|
|
- func: "save bazel exec logs"
|
|
- func: "save hang analyzer debugger files"
|
|
- func: "save disk statistics"
|
|
- func: "save system resource information"
|
|
- func: "save libfuzzertest corpora"
|
|
- func: "remove files"
|
|
vars:
|
|
files: >-
|
|
src/resmoke_error_code
|
|
src/*.gcda.gcov
|
|
src/gcov-intermediate-files.tgz
|
|
src/*.core src/*.mdmp src/*.core.gz src/*.mdmp.gz
|
|
mongo-coredumps.json
|
|
src/dist-unittests/bin/*
|
|
src/dist-unittests/lib/*
|
|
mongo-unittests.tgz
|
|
src/debugger*.*
|
|
src/mongo-hanganalyzer.tgz
|
|
diskstats.tgz
|
|
system-resource-info.tgz
|
|
${report_file|src/report.json}
|
|
${archive_file|src/archive.json}
|
|
setup_group_can_fail_task: true
|
|
setup_group:
|
|
- command: manifest.load
|
|
- func: "git get project and add git tag"
|
|
- func: "set task expansion macros"
|
|
- func: "f_expansions_write"
|
|
- func: "kill processes"
|
|
- func: "cleanup environment"
|
|
# The python virtual environment is installed in ${workdir}, which is created in
|
|
# "set up venv".
|
|
- func: "set up venv"
|
|
- func: "upload pip requirements"
|
|
- func: "f_expansions_write"
|
|
- func: "configure evergreen api credentials"
|
|
- func: "get buildnumber"
|
|
- func: "f_expansions_write"
|
|
- func: "set up credentials"
|
|
- func: "f_expansions_write"
|
|
- func: "use WiredTiger develop" # noop if ${use_wt_develop} is not "true"
|
|
- func: "f_expansions_write"
|
|
teardown_group:
|
|
- func: "f_expansions_write"
|
|
- func: "cleanup environment"
|
|
timeout:
|
|
- func: "f_expansions_write"
|
|
- func: "run hang analyzer"
|
|
- func: "wait for resmoke to shutdown"
|
|
|
|
# TODO: split this up into the user files.
|
|
# List of all variants that make mongocryptd
|
|
# If a variant is listed here and has a push task, mongocryptd is pushed
|
|
- mongocryptd_variants: &mongocryptd_variants
|
|
- enterprise-amazon2023
|
|
# TODO(SERVER-85904) - enterprise-amazon2023-lto
|
|
- enterprise-amazon2023-arm64
|
|
- enterprise-amazon2023-arm64-grav4
|
|
# TODO(SERVER-85904) - enterprise-amazon2023-arm64-lto
|
|
- enterprise-debian12-64
|
|
- enterprise-linux-64-amazon-ami
|
|
- enterprise-macos
|
|
- enterprise-macos-arm64
|
|
- enterprise-rhel-81-ppc64le
|
|
- enterprise-rhel-8-64-bit
|
|
- enterprise-rhel-8-64-bit-coverage
|
|
- enterprise-rhel-8-64-bit-suggested
|
|
- enterprise-rhel-8-arm64
|
|
- enterprise-rhel-83-s390x
|
|
- enterprise-rhel-9-ppc64le
|
|
- enterprise-rhel-90-64-bit
|
|
- enterprise-rhel-90-arm64
|
|
- enterprise-rhel-93-64-bit
|
|
- enterprise-rhel-93-arm64
|
|
- enterprise-suse15-64
|
|
- enterprise-ubuntu2004-arm64
|
|
- enterprise-ubuntu2204-arm64
|
|
- enterprise-ubuntu2404-arm64
|
|
- enterprise-ubuntu2004-64
|
|
- enterprise-ubuntu2204-64
|
|
- enterprise-ubuntu2404
|
|
- enterprise-windows
|
|
- enterprise-windows-debug-unoptimized
|
|
- enterprise-windows-inmem
|
|
- enterprise-windows-wtdevelop
|
|
|
|
################################################
|
|
# Tasks #
|
|
################################################
|
|
tasks:
|
|
# TODO(SERVER-82195): simplify this to avoid duplication of parameters in other tests.
|
|
- name: compile_dist_test_quick
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: >-
|
|
//src/mongo/db/commands:fsync_locked_with_debug
|
|
bazel_args: >-
|
|
--config=evg
|
|
- func: "verify build output present"
|
|
vars:
|
|
output: bazel-bin/src/mongo/db/commands/libfsync_locked_with_debug.lo
|
|
|
|
- name: run_bazel_program
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "bazel run"
|
|
vars:
|
|
target: >-
|
|
//src/mongo/platform:visibility1_test
|
|
|
|
- name: bazel_archive_and_run_dist_test
|
|
tags: ["assigned_to_jira_team_devprod_build", "bazel_check", "auxiliary"]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
exec_timeout_secs: 32400 # 9 hours
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: archive-dist-test
|
|
bazel_args: --config=opt --separate_debug=True
|
|
- func: "run tests"
|
|
vars:
|
|
suite: core
|
|
install_dir: bazel-bin/install/bin
|
|
resmoke_jobs_max: 4 # No cap on number of jobs.
|
|
exec_timeout_secs: 32400 # 9 hours
|
|
|
|
- name: hydrate_bazel_profile_fastbuild
|
|
tags:
|
|
[
|
|
"assigned_to_jira_team_devprod_build",
|
|
"bazel_cache_hydration",
|
|
"auxiliary",
|
|
]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: //src/...
|
|
# Use --linkopt="-s" to dramatically reduce link time, since we only care about hydrating cache
|
|
# on CppCompile as CppLink is performed locally and not cached remotely.
|
|
bazel_args: >-
|
|
--config=fastbuild
|
|
--define GIT_COMMIT_HASH=nogitversion
|
|
--output_groups=compilation_outputs
|
|
--keep_going
|
|
--build_tag_filters=${bazel_filters_for_cache_hydration}
|
|
|
|
- name: hydrate_required_bazel_profile_opt
|
|
tags:
|
|
[
|
|
"assigned_to_jira_team_devprod_build",
|
|
"bazel_cache_hydration",
|
|
"auxiliary",
|
|
]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: //src/...
|
|
# TODO(SERVER-99213): limit to --jobs 32 once fission is enabled by default in CI
|
|
bazel_args: >-
|
|
--config=opt
|
|
--define GIT_COMMIT_HASH=nogitversion
|
|
--output_groups=compilation_outputs
|
|
--keep_going
|
|
--build_tag_filters=${bazel_filters_for_cache_hydration}
|
|
|
|
- name: hydrate_bazel_profile_dbg
|
|
tags:
|
|
[
|
|
"assigned_to_jira_team_devprod_build",
|
|
"bazel_cache_hydration",
|
|
"auxiliary",
|
|
]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: //src/...
|
|
# Use --linkopt="-s" to dramatically reduce link time, since we only care about hydrating cache
|
|
# on CppCompile as CppLink is performed locally and not cached remotely.
|
|
bazel_args: >-
|
|
--config=dbg
|
|
--define GIT_COMMIT_HASH=nogitversion
|
|
--output_groups=compilation_outputs
|
|
--keep_going
|
|
--build_tag_filters=${bazel_filters_for_cache_hydration}
|
|
|
|
- name: hydrate_bazel_profile_dbg_aubsan
|
|
tags:
|
|
[
|
|
"assigned_to_jira_team_devprod_build",
|
|
"bazel_cache_hydration",
|
|
"auxiliary",
|
|
]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: //src/...
|
|
# Use --linkopt="-s" to dramatically reduce link time, since we only care about hydrating cache
|
|
# on CppCompile as CppLink is performed locally and not cached remotely.
|
|
bazel_args: >-
|
|
--config=dbg_aubsan
|
|
--define GIT_COMMIT_HASH=nogitversion
|
|
--output_groups=compilation_outputs
|
|
--keep_going
|
|
--build_tag_filters=${bazel_filters_for_cache_hydration}
|
|
|
|
- name: hydrate_bazel_profile_dbg_tsan
|
|
tags:
|
|
[
|
|
"assigned_to_jira_team_devprod_build",
|
|
"bazel_cache_hydration",
|
|
"auxiliary",
|
|
]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: //src/...
|
|
# Use --linkopt="-s" to dramatically reduce link time, since we only care about hydrating cache
|
|
# on CppCompile as CppLink is performed locally and not cached remotely.
|
|
bazel_args: >-
|
|
--config=dbg_tsan
|
|
--define GIT_COMMIT_HASH=nogitversion
|
|
--output_groups=compilation_outputs
|
|
--keep_going
|
|
--build_tag_filters=${bazel_filters_for_cache_hydration}
|
|
|
|
- name: run_bazel_clang_tidy
|
|
tags:
|
|
[
|
|
"assigned_to_jira_team_devprod_build",
|
|
"development_critical_single_variant",
|
|
"requires_large_host",
|
|
"clang_tidy",
|
|
]
|
|
exec_timeout_secs: 1800 # 30 min timeout for the task overall
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "do bazel setup"
|
|
- func: "bazel compile"
|
|
vars:
|
|
# Note that we only run clang-tidy on the Mongo-authored targets, not on third-party code.
|
|
targets: //src/mongo/...
|
|
bazel_args: --config=clang-tidy --mongo_toolchain_version=${clang_tidy_toolchain|v5} --keep_going
|
|
- func: "generate clang-tidy report"
|
|
|
|
- name: run_bazel_compiledb
|
|
tags:
|
|
[
|
|
"assigned_to_jira_team_devprod_build",
|
|
"development_critical_single_variant",
|
|
"lint",
|
|
]
|
|
exec_timeout_secs: 600 # 10 min timeout for the task overall
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "do bazel setup"
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: compiledb
|
|
- func: bazel run
|
|
vars:
|
|
target: //evergreen:validate_compile_commands
|
|
|
|
- name: run_bazel_program_windows
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "bazel run"
|
|
vars:
|
|
target: >-
|
|
//src/mongo/platform:visibility1_test
|
|
# TODO DEVPROD-2508 remove dbg flags when compilation_mode is synced with build_mode automatically
|
|
args: >-
|
|
--compilation_mode=dbg
|
|
|
|
- name: compile_upload_benchmarks
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: >-
|
|
archive-mongo_benchmark-stripped
|
|
bazel_args: >-
|
|
--config=evg
|
|
--include_autogenerated_targets=True
|
|
--build_tag_filters=-repl_bm,-query_bm,-bsoncolumn_bm,-first_half_bm,-second_half_bm,-storage_bm,-sharding_bm,-sep_bm
|
|
compiling_for_test: true
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/bazel-bin/mongo_benchmark-stripped.${ext|tgz}
|
|
remote_file: ${mongo_benchmarks}
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: application/tar
|
|
display_name: Benchmarks
|
|
|
|
- name: compile_upload_benchmarks_debug
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
# TODO(SERVER-86426): Figure out a workable long-term solution to benchmark debug symbols
|
|
disable: true
|
|
depends_on:
|
|
- name: compile_upload_benchmarks
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: >-
|
|
archive-mongo_benchmark-debug
|
|
bazel_args: >-
|
|
--config=evg
|
|
--include_autogenerated_targets=True
|
|
--build_tag_filters=-repl_bm,-query_bm,-bsoncolumn_bm,-first_half_bm,-second_half_bm,-storage_bm,-sharding_bm,-sep_bm
|
|
compiling_for_test: true
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/bazel-bin/mongo_benchmark-debug.${ext|tgz}
|
|
remote_file: ${mongo_benchmarks_debugsymbols}
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: application/tar
|
|
display_name: Benchmarks Debug
|
|
|
|
## compile_all - build all bazel targets ##
|
|
- name: compile_all_future_git_tag_multiversion
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
depends_on:
|
|
- name: version_expansions_future_git_tag_multiversion_gen
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: //src/...
|
|
bazel_args: >-
|
|
--config=evg
|
|
compiling_for_test: true
|
|
|
|
- name: compile_all_but_not_unittests
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
depends_on:
|
|
- name: archive_dist_test
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: //src/...
|
|
compiling_for_test: true
|
|
# The executables generated by the install-all target are not used after successfully being
|
|
# compiled. We therefore link them without debug symbols to compile them faster.
|
|
bazel_args: >-
|
|
--config=evg
|
|
--build_tag_filters=-mongo_unittest,mongo_binary,mongo_benchmark,mongo_integration_test
|
|
--linkopt=-s
|
|
${compile_all_but_not_unittests_flags|--linkopt=-s}
|
|
|
|
## compile_unittests ##
|
|
- name: compile_unittests_future_git_tag_multiversion
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
depends_on:
|
|
- name: version_expansions_future_git_tag_multiversion_gen
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: install-mongo_unittest
|
|
bazel_args: >-
|
|
--config=evg
|
|
--include_autogenerated_targets=True
|
|
compiling_for_test: true
|
|
|
|
- name: compile_libraries_for_unittests
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
exec_timeout_secs: 32400 # 9 hours
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
# Target the conf to just run the bazel part of the build
|
|
targets: //src/...
|
|
compiling_for_test: true
|
|
task_compile_flags: ${unittest_library_compile_flags}
|
|
bazel_args: >-
|
|
--config=evg
|
|
--build_tag_filters=mongo_library
|
|
|
|
## run_unittests ##
|
|
- name: run_unittests_future_git_tag_multiversion
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
depends_on:
|
|
- name: compile_unittests_future_git_tag_multiversion
|
|
commands:
|
|
- func: "f_expansions_write"
|
|
- func: "run diskstats"
|
|
- func: "f_expansions_write"
|
|
- func: "monitor process threads"
|
|
- func: "collect system resource info"
|
|
- func: "run tests"
|
|
vars:
|
|
suite: unittests
|
|
install_dir: bazel-bin/install/bin
|
|
|
|
##compile_and_archive_libfuzzertests - build libfuzzertests ##
|
|
- name: compile_and_archive_libfuzzertests
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: archive-mongo_fuzzer_test
|
|
compiling_for_test: true
|
|
bazel_args: >-
|
|
--config=evg
|
|
--include_autogenerated_targets=True
|
|
# Store the fuzzer executable, which we use to generate and run fuzzer inputs.
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: "src/bazel-bin/mongo_fuzzer_test.${ext|tgz}"
|
|
remote_file: "${project}/libfuzzer-tests/${build_variant}/${revision}/libfuzzer-tests.tgz"
|
|
bucket: mciuploads
|
|
permissions: private
|
|
visibility: signed
|
|
content_type: application/tar
|
|
display_name: "LibFuzzer Tests"
|
|
|
|
## compile_dbtest ##
|
|
- &compile_dbtest
|
|
name: compile_dbtest
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
depends_on:
|
|
- name: archive_dist_test
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: install-dbtest
|
|
bazel_args: >-
|
|
--config=evg
|
|
|
|
- <<: *compile_dbtest
|
|
name: compile_dbtest_future_git_tag_multiversion
|
|
depends_on:
|
|
- name: version_expansions_future_git_tag_multiversion_gen
|
|
|
|
## run_dbtest ##
|
|
- &run_dbtest
|
|
name: run_dbtest
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"]
|
|
depends_on:
|
|
- name: compile_dbtest
|
|
commands:
|
|
- func: "f_expansions_write"
|
|
- func: "run diskstats"
|
|
- func: "f_expansions_write"
|
|
- func: "monitor process threads"
|
|
- func: "collect system resource info"
|
|
- func: "run tests"
|
|
vars:
|
|
suite: dbtest
|
|
install_dir: bazel-bin/install/bin
|
|
|
|
- <<: *run_dbtest
|
|
name: run_dbtest_future_git_tag_multiversion
|
|
depends_on:
|
|
- name: compile_dbtest_future_git_tag_multiversion
|
|
|
|
- &archive_dbtest
|
|
name: archive_dbtest
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"]
|
|
depends_on:
|
|
- name: compile_dbtest
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: archive-dbtest
|
|
bazel_args: >-
|
|
--config=evg
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/bazel-bin/dbtest.${ext|tgz}
|
|
remote_file: ${project}/${build_variant}/${revision}/dbtest/dbtest-${build_id}-${task_name}-${execution}.tgz
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: application/tar
|
|
display_name: dbtest binary - Execution ${execution}
|
|
optional: true
|
|
|
|
- <<: *archive_dbtest
|
|
name: archive_dbtest_future_git_tag_multiversion
|
|
depends_on:
|
|
- name: compile_dbtest_future_git_tag_multiversion
|
|
|
|
- name: compile_integration_test
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
depends_on:
|
|
- name: archive_dist_test
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: install-mongo_integration_test
|
|
bazel_args: >-
|
|
--config=evg
|
|
--include_autogenerated_targets=True
|
|
task_compile_flags: ${integration_test_compile_flags}
|
|
compiling_for_test: true
|
|
|
|
- name: archive_jstestshell
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "activate task"
|
|
vars:
|
|
task_to_activate: archive_jstestshell_debug
|
|
- func: "bazel compile"
|
|
vars: &archive_jstestshell_vars
|
|
targets: archive-mongo-stripped
|
|
bazel_args: >-
|
|
--config=evg
|
|
task_compile_flags: >-
|
|
--asan=False
|
|
--tsan=False
|
|
--ubsan=False
|
|
--linkstatic=True
|
|
--dbg=True
|
|
--opt=on
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/bazel-bin/mongo-stripped.${ext|tgz}
|
|
remote_file: ${mongo_jstestshell}
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: ${content_type|application/gzip}
|
|
display_name: Jstestshell
|
|
|
|
- name: archive_jstestshell_debug
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
depends_on:
|
|
- name: archive_jstestshell
|
|
commands:
|
|
- func: "f_expansions_write"
|
|
- func: "bazel compile"
|
|
vars:
|
|
<<: *archive_jstestshell_vars
|
|
targets: archive-mongo-debug
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/bazel-bin/mongo-debug.${ext|tgz}
|
|
remote_file: ${mongo_jstestshell_debugsymbols}
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: ${content_type|application/gzip}
|
|
display_name: Jstestshell Debugsymbols
|
|
|
|
- name: stitch_support_create_lib
|
|
tags:
|
|
[
|
|
"assigned_to_jira_team_devprod_build",
|
|
"release_critical",
|
|
"requires_compile_variant",
|
|
"requires_large_host",
|
|
"stitch",
|
|
"bazel_check",
|
|
]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "do bazel setup"
|
|
- func: "f_expansions_write"
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: archive-stitch_support
|
|
bazel_args: >-
|
|
--config=evg_stitch
|
|
task_compile_flags: --linkstatic=True
|
|
- func: "f_expansions_write"
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: "src/bazel-bin/stitch_support.${ext|tgz}"
|
|
remote_file: "${project}/stitch-support/${build_variant}/${revision}/stitch-support-${version}.tgz"
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: application/tar
|
|
display_name: "Stitch Support Library"
|
|
|
|
- name: stitch_support_install_and_run_tests
|
|
tags:
|
|
[
|
|
"assigned_to_jira_team_devprod_build",
|
|
"release_critical",
|
|
"incompatible_mac",
|
|
"requires_compile_variant",
|
|
"requires_large_host",
|
|
"stitch",
|
|
"bazel_check",
|
|
]
|
|
depends_on:
|
|
- name: stitch_support_create_lib
|
|
commands:
|
|
- func: "do bazel setup"
|
|
- func: "f_expansions_write"
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: install-stitch_support_test
|
|
compiling_for_test: true
|
|
bazel_args: >-
|
|
--config=evg_stitch_test
|
|
- func: "get and apply version expansions"
|
|
- func: "f_expansions_write"
|
|
- command: subprocess.exec
|
|
type: test
|
|
params:
|
|
binary: bash
|
|
args:
|
|
- "src/evergreen/stitch_support_run_tests.sh"
|
|
|
|
- name: crypt_create_debug_lib
|
|
tags: ["assigned_to_jira_team_server_security", "auxiliary", "bazel_check"]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "f_expansions_write"
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: archive-mongo_crypt-debug archive-mongo_crypt-stripped
|
|
bazel_args: >-
|
|
--config=evg_crypt_dbg
|
|
task_compile_flags: --linkstatic=True
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: "src/bazel-bin/mongo_crypt-stripped.${ext|tgz}"
|
|
remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shared_v1_dev-${version}.${ext|tgz}"
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: ${content_type|application/tar}
|
|
display_name: "Mongo Crypt Library dev"
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: "src/bazel-bin/mongo_crypt-debug.${ext|tgz}"
|
|
remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shared_v1_debug-${version}.${ext|tgz}"
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: ${content_type|application/tar}
|
|
display_name: "Mongo Crypt Library debug"
|
|
|
|
- name: crypt_install_tests
|
|
tags: ["assigned_to_jira_team_server_security", "auxiliary", "bazel_check"]
|
|
depends_on:
|
|
- name: crypt_create_debug_lib
|
|
commands:
|
|
- func: "f_expansions_write"
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: archive-mongo_crypt_shlib_test-stripped
|
|
compiling_for_test: true
|
|
bazel_args: >-
|
|
--config=evg_crypt_test
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: "src/bazel-bin/mongo_crypt_shlib_test-stripped.${ext|tgz}"
|
|
remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shlib_test-${version}.${ext|tgz}"
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: ${content_type|application/tar}
|
|
display_name: "Mongo Crypt Shared Library Test"
|
|
|
|
- name: crypt_run_tests
|
|
tags: ["assigned_to_jira_team_server_security", "auxiliary", "bazel_check"]
|
|
depends_on:
|
|
- name: crypt_install_tests
|
|
commands:
|
|
- func: "get and apply version expansions"
|
|
- func: "f_expansions_write"
|
|
- command: subprocess.exec
|
|
type: test
|
|
params:
|
|
binary: bash
|
|
args:
|
|
- "src/evergreen/crypt_run_tests.sh"
|
|
|
|
- name: publish_packages
|
|
run_on: rhel8.7-small
|
|
tags:
|
|
[
|
|
"assigned_to_jira_team_devprod_build",
|
|
"release_critical",
|
|
"incompatible_development_variant",
|
|
"requires_compile_variant",
|
|
"publish",
|
|
]
|
|
# This should prevent this task from running in patch builds, where we
|
|
# don't want to publish packages.
|
|
patchable: false
|
|
stepback: false
|
|
# Same dependencies as "push" below
|
|
depends_on:
|
|
- name: package
|
|
- name: jsCore
|
|
- name: run_dbtest
|
|
- name: replica_sets_jscore_passthrough_gen
|
|
commands:
|
|
- command: manifest.load
|
|
- func: "git get project and add git tag"
|
|
- func: "get and apply version expansions"
|
|
- func: "f_expansions_write"
|
|
- func: "kill processes"
|
|
- func: "cleanup environment"
|
|
- func: "set up venv"
|
|
- func: "fetch packages"
|
|
- func: "f_expansions_write"
|
|
- func: "set up remote credentials"
|
|
vars:
|
|
aws_key_remote: ${repo_aws_key}
|
|
aws_secret_remote: ${repo_aws_secret}
|
|
- func: "set up notary client credentials"
|
|
- func: "f_expansions_write"
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
silent: true
|
|
args:
|
|
- "./src/evergreen/container_registry_login.sh"
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${upload_lock_access_key_id}
|
|
AWS_SECRET_ACCESS_KEY: ${upload_lock_secret_access_key}
|
|
UPLOAD_LOCK_IMAGE: ${upload_lock_image}
|
|
UPLOAD_BUCKET: ${upload_lock_bucket}
|
|
AWS_REGION: ${upload_lock_region}
|
|
EVERGREEN_TASK_ID: ${task_id}
|
|
args:
|
|
- "./src/evergreen/packages_publish.sh"
|
|
|
|
- name: push
|
|
run_on: rhel8.7-small
|
|
tags:
|
|
[
|
|
"assigned_to_jira_team_devprod_build",
|
|
"release_critical",
|
|
"incompatible_development_variant",
|
|
"requires_compile_variant",
|
|
"publish",
|
|
]
|
|
patchable: false
|
|
depends_on:
|
|
- name: package
|
|
- name: jsCore
|
|
- name: run_dbtest
|
|
- name: replica_sets_jscore_passthrough_gen
|
|
stepback: false
|
|
commands:
|
|
- command: manifest.load
|
|
- func: "git get project and add git tag"
|
|
- func: "get and apply version expansions"
|
|
- func: "f_expansions_write"
|
|
- func: "kill processes"
|
|
- func: "cleanup environment"
|
|
- func: "set up venv"
|
|
- func: "fetch packages"
|
|
- func: "fetch dist tarball"
|
|
# Fetch mongocryptd
|
|
- command: s3.get
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
remote_file: ${mongo_cryptd}
|
|
bucket: mciuploads
|
|
local_file: src/mongo-cryptd.tgz
|
|
build_variants: *mongocryptd_variants
|
|
- func: "f_expansions_write"
|
|
- func: "fetch dist debugsymbols"
|
|
- func: "set up remote credentials"
|
|
vars:
|
|
aws_key_remote: ${repo_aws_key}
|
|
aws_secret_remote: ${repo_aws_secret}
|
|
- func: "f_expansions_write"
|
|
|
|
# login to container registry
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
silent: true
|
|
args:
|
|
- "./src/evergreen/container_registry_login.sh"
|
|
|
|
# signing windows artifacts
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
args:
|
|
- "./src/evergreen/garasign_jsign_sign.sh"
|
|
|
|
# signing linux artifacts
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
args:
|
|
- "./src/evergreen/garasign_gpg_sign.sh"
|
|
|
|
# Put the binaries tarball/zipfile
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}
|
|
aws_key: ${aws_key}
|
|
bucket: ${push_bucket}
|
|
permissions: public-read
|
|
content_type: ${content_type|application/gzip}
|
|
remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}
|
|
skip_existing: ${is_release}
|
|
# Put the cryptd tarball/zipfile
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}
|
|
aws_key: ${aws_key}
|
|
bucket: ${push_bucket}
|
|
permissions: public-read
|
|
content_type: ${content_type|application/gzip}
|
|
remote_file: ${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}
|
|
skip_existing: ${is_release}
|
|
build_variants: *mongocryptd_variants
|
|
|
|
# Put the debug symbols
|
|
# push directly to repo due to limitations in file size SERVER-63432
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
aws_key: ${aws_key}
|
|
permissions: public-read
|
|
local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}
|
|
bucket: ${push_bucket}
|
|
content_type: ${content_type|application/gzip}
|
|
remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}
|
|
skip_existing: ${is_release}
|
|
optional: true
|
|
|
|
# Put the binaries tarball signature
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig
|
|
aws_key: ${aws_key}
|
|
bucket: ${push_bucket}
|
|
permissions: public-read
|
|
content_type: ${content_type|application/gzip}
|
|
remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig
|
|
skip_existing: ${is_release}
|
|
|
|
# Put the cryptd tarball signature
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig
|
|
aws_key: ${aws_key}
|
|
bucket: ${push_bucket}
|
|
permissions: public-read
|
|
content_type: ${content_type|application/gzip}
|
|
remote_file: ${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig
|
|
skip_existing: ${is_release}
|
|
build_variants: *mongocryptd_variants
|
|
|
|
# Put the debug symbols signature
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
aws_key: ${aws_key}
|
|
permissions: public-read
|
|
local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sig
|
|
bucket: ${push_bucket}
|
|
content_type: ${content_type|application/gzip}
|
|
remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sig
|
|
skip_existing: ${is_release}
|
|
optional: true
|
|
|
|
# Put the signed MSI file
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
permissions: public-read
|
|
build_variants: ["enterprise-windows", "windows"]
|
|
local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi
|
|
bucket: ${push_bucket}
|
|
content_type: application/x-msi
|
|
remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi
|
|
skip_existing: ${is_release}
|
|
|
|
# Put the binaries tarball sha1
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1
|
|
aws_key: ${aws_key}
|
|
permissions: public-read
|
|
bucket: ${push_bucket}
|
|
content_type: text/plain
|
|
remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1
|
|
skip_existing: ${is_release}
|
|
|
|
# Put the cryptd tarball sha1
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1
|
|
aws_key: ${aws_key}
|
|
permissions: public-read
|
|
bucket: ${push_bucket}
|
|
content_type: text/plain
|
|
remote_file: ${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1
|
|
skip_existing: ${is_release}
|
|
build_variants: *mongocryptd_variants
|
|
|
|
# Put the debug symbols sha1
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
aws_key: ${aws_key}
|
|
permissions: public-read
|
|
local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha1
|
|
bucket: ${push_bucket}
|
|
content_type: text/plain
|
|
remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha1
|
|
skip_existing: ${is_release}
|
|
optional: true
|
|
|
|
# Push the signed MSI sha1
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
permissions: public-read
|
|
build_variants: ["enterprise-windows", "windows"]
|
|
local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.sha1
|
|
bucket: ${push_bucket}
|
|
content_type: text/plain
|
|
remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha1
|
|
skip_existing: ${is_release}
|
|
|
|
# Put the binaries tarball sha256
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256
|
|
permissions: public-read
|
|
aws_key: ${aws_key}
|
|
bucket: ${push_bucket}
|
|
content_type: text/plain
|
|
remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256
|
|
skip_existing: ${is_release}
|
|
|
|
# Put the cryptd tarball sha256
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256
|
|
permissions: public-read
|
|
aws_key: ${aws_key}
|
|
bucket: ${push_bucket}
|
|
content_type: text/plain
|
|
remote_file: ${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256
|
|
skip_existing: ${is_release}
|
|
build_variants: *mongocryptd_variants
|
|
|
|
# Put the debug symbols sha256
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha256
|
|
aws_key: ${aws_key}
|
|
bucket: ${push_bucket}
|
|
permissions: public-read
|
|
content_type: text/plain
|
|
remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha256
|
|
skip_existing: ${is_release}
|
|
optional: true
|
|
|
|
# Put the signed MSI sha256
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
build_variants: ["enterprise-windows", "windows"]
|
|
local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.sha256
|
|
bucket: ${push_bucket}
|
|
permissions: public-read
|
|
remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha256
|
|
skip_existing: ${is_release}
|
|
content_type: text/plain
|
|
|
|
# Put the binaries tarball md5
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5
|
|
aws_key: ${aws_key}
|
|
bucket: ${push_bucket}
|
|
permissions: public-read
|
|
content_type: text/plain
|
|
remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5
|
|
skip_existing: ${is_release}
|
|
|
|
# Put the cryptd tarball md5
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5
|
|
aws_key: ${aws_key}
|
|
bucket: ${push_bucket}
|
|
permissions: public-read
|
|
content_type: text/plain
|
|
remote_file: ${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5
|
|
skip_existing: ${is_release}
|
|
build_variants: *mongocryptd_variants
|
|
|
|
# Put the debug symbols md5
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.md5
|
|
bucket: ${push_bucket}
|
|
content_type: text/plain
|
|
permissions: public-read
|
|
remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.md5
|
|
skip_existing: ${is_release}
|
|
optional: true
|
|
|
|
# Put the signed MSI md5
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
build_variants: ["enterprise-windows", "windows"]
|
|
local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.md5
|
|
bucket: ${push_bucket}
|
|
permissions: public-read
|
|
content_type: text/plain
|
|
remote_file: ${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.md5
|
|
skip_existing: ${is_release}
|
|
|
|
- command: subprocess.exec
|
|
params:
|
|
continue_on_err: true
|
|
binary: bash
|
|
env:
|
|
SERVER_TARBALL_PATH: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}
|
|
SERVER_TARBALL_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}
|
|
CRYPTD_TARBALL_PATH: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}
|
|
CRYPTD_TARBALL_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}
|
|
SOURCE_TARBALL_PATH: src/mongodb-src-${src_suffix}.${ext|tar.gz}
|
|
SOURCE_TARBALL_KEY: ${version_id}/${build_id}/push/src/mongodb-src-${src_suffix}.${ext|tar.gz}
|
|
DEBUG_SYMBOLS_TARBALL_PATH: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}
|
|
DEBUG_SYMBOLS_TARBALL_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}
|
|
SERVER_TARBALL_SIGNATURE_PATH: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig
|
|
SERVER_TARBALL_SIGNATURE_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig
|
|
CRYPTD_TARBALL_SIGNATURE_PATH: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig
|
|
CRYPTD_TARBALL_SIGNATURE_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig
|
|
SOURCE_TARBALL_SIGNATURE_PATH: src/mongodb-src-${src_suffix}.${ext|tar.gz}.sig
|
|
SOURCE_TARBALL_SIGNATURE_KEY: ${version_id}/${build_id}/push/src/mongodb-src-${src_suffix}.${ext|tar.gz}.sig
|
|
DEBUG_SYMBOLS_TARBALL_SIGNATURE_PATH: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sig
|
|
DEBUG_SYMBOLS_TARBALL_SIGNATURE_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sig
|
|
MSI_PATH: src/mongodb-${push_name}-${push_arch}-${suffix}.msi
|
|
MSI_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi
|
|
SERVER_TARBALL_SHA1_PATH: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1
|
|
SERVER_TARBALL_SHA1_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1
|
|
CRYPTD_TARBALL_SHA1_PATH: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1
|
|
CRYPTD_TARBALL_SHA1_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1
|
|
SOURCE_TARBALL_SHA1_PATH: src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha1
|
|
SOURCE_TARBALL_SHA1_KEY: ${version_id}/${build_id}/push/src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha1
|
|
DEBUG_SYMBOLS_TARBALL_SHA1_PATH: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha1
|
|
DEBUG_SYMBOLS_TARBALL_SHA1_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha1
|
|
MSI_SHA1_PATH: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.sha1
|
|
MSI_SHA1_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha1
|
|
SERVER_TARBALL_SHA256_PATH: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256
|
|
SERVER_TARBALL_SHA256_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256
|
|
CRYPTD_TARBALL_SHA256_PATH: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256
|
|
CRYPTD_TARBALL_SHA256_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256
|
|
SOURCE_TARBALL_SHA256_PATH: src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha256
|
|
SOURCE_TARBALL_SHA256_KEY: ${version_id}/${build_id}/push/src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha256
|
|
DEBUG_SYMBOLS_TARBALL_SHA256_PATH: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha256
|
|
DEBUG_SYMBOLS_TARBALL_SHA256_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha256
|
|
MSI_SHA256_PATH: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.sha256
|
|
MSI_SHA256_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha256
|
|
SERVER_TARBALL_MD5_PATH: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5
|
|
SERVER_TARBALL_MD5_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5
|
|
CRYPTD_TARBALL_MD5_PATH: src/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5
|
|
CRYPTD_TARBALL_MD5_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5
|
|
SOURCE_TARBALL_MD5_PATH: src/mongodb-src-${src_suffix}.${ext|tar.gz}.md5
|
|
SOURCE_TARBALL_MD5_KEY: ${version_id}/${build_id}/push/src/mongodb-src-${src_suffix}.${ext|tar.gz}.md5
|
|
DEBUG_SYMBOLS_TARBALL_MD5_PATH: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.md5
|
|
DEBUG_SYMBOLS_TARBALL_MD5_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.md5
|
|
MSI_MD5_PATH: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.md5
|
|
MSI_MD5_KEY: ${version_id}/${build_id}/push/${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.md5
|
|
AWS_ACCESS_KEY_ID: ${upload_lock_access_key_id}
|
|
AWS_SECRET_ACCESS_KEY: ${upload_lock_secret_access_key}
|
|
UPLOAD_LOCK_IMAGE: ${upload_lock_image}
|
|
UPLOAD_BUCKET: ${upload_lock_bucket}
|
|
AWS_REGION: ${upload_lock_region}
|
|
EVERGREEN_TASK_ID: ${task_id}
|
|
args:
|
|
- "./src/evergreen/run_upload_lock_push.sh"
|
|
#Trace artifacts (binaries, cryptd, debug, MSI) using Papertrail
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
env:
|
|
IS_RELEASE: ${is_release}
|
|
args:
|
|
- "./src/evergreen/papertrail_generate_expansions.sh"
|
|
- command: expansions.update
|
|
params:
|
|
file: src/papertrail-expansions.yml
|
|
- func: "f_expansions_write"
|
|
- command: papertrail.trace
|
|
params:
|
|
key_id: ${papertrail_key_id}
|
|
secret_key: ${papertrail_secret_key}
|
|
product: ${product_for_papertrail}
|
|
version: ${release_version}
|
|
filenames:
|
|
- "src/mongodb-${push_name}-${push_arch}*"
|
|
- "src/mongodb-cryptd-${push_name}-${push_arch}*"
|
|
|
|
- name: crypt_push
|
|
run_on: rhel8.7-small
|
|
tags:
|
|
[
|
|
"assigned_to_jira_team_server_security",
|
|
"release_critical",
|
|
"incompatible_development_variant",
|
|
"incompatible_community",
|
|
"requires_compile_variant",
|
|
"publish_crypt",
|
|
]
|
|
patchable: false
|
|
stepback: false
|
|
depends_on:
|
|
- name: crypt_create_lib
|
|
commands:
|
|
- command: manifest.load
|
|
- func: "f_expansions_write"
|
|
- func: "git get project and add git tag"
|
|
- func: "set task expansion macros"
|
|
- func: "f_expansions_write"
|
|
- func: "kill processes"
|
|
- func: "cleanup environment"
|
|
- func: "set up venv"
|
|
- func: "upload pip requirements"
|
|
- func: "get buildnumber"
|
|
- func: "f_expansions_write"
|
|
- func: "get and apply version expansions"
|
|
- func: "f_expansions_write"
|
|
- command: s3.get
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
remote_file: ${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shared_v1-${version}.${ext|tgz}
|
|
bucket: mciuploads
|
|
local_file: src/mongo_crypt_shared_v1.${ext|tgz}
|
|
- func: "f_expansions_write"
|
|
- func: "set up remote credentials"
|
|
vars:
|
|
aws_key_remote: ${repo_aws_key}
|
|
aws_secret_remote: ${repo_aws_secret}
|
|
- func: "f_expansions_write"
|
|
# login to container registry
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
silent: true
|
|
args:
|
|
- "./src/evergreen/container_registry_login.sh"
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
args:
|
|
- "./src/evergreen/garasign_gpg_crypt_sign.sh"
|
|
# Put the crypt tarball/zipfile
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}
|
|
aws_key: ${aws_key}
|
|
bucket: ${push_bucket}
|
|
permissions: public-read
|
|
content_type: ${content_type|application/gzip}
|
|
remote_file: ${push_path}/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}
|
|
skip_existing: ${is_release}
|
|
# Put the crypt tarball signature
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig
|
|
aws_key: ${aws_key}
|
|
bucket: ${push_bucket}
|
|
permissions: public-read
|
|
content_type: ${content_type|application/gzip}
|
|
remote_file: ${push_path}/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig
|
|
skip_existing: ${is_release}
|
|
# Put the crypt tarball sha1
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1
|
|
aws_key: ${aws_key}
|
|
permissions: public-read
|
|
bucket: ${push_bucket}
|
|
content_type: text/plain
|
|
remote_file: ${push_path}/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1
|
|
skip_existing: ${is_release}
|
|
# Put the crypt tarball sha256
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256
|
|
permissions: public-read
|
|
aws_key: ${aws_key}
|
|
bucket: ${push_bucket}
|
|
content_type: text/plain
|
|
remote_file: ${push_path}/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256
|
|
skip_existing: ${is_release}
|
|
# Put the crypt tarball md5
|
|
- command: s3.put
|
|
params:
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5
|
|
aws_key: ${aws_key}
|
|
bucket: ${push_bucket}
|
|
permissions: public-read
|
|
content_type: text/plain
|
|
remote_file: ${push_path}/mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5
|
|
skip_existing: ${is_release}
|
|
#Trace artifacts (crypt) using Papertrail
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
env:
|
|
IS_RELEASE: ${is_release}
|
|
args:
|
|
- "./src/evergreen/papertrail_generate_expansions.sh"
|
|
- command: expansions.update
|
|
params:
|
|
file: src/papertrail-expansions.yml
|
|
- func: "f_expansions_write"
|
|
- command: papertrail.trace
|
|
params:
|
|
key_id: ${papertrail_key_id}
|
|
secret_key: ${papertrail_secret_key}
|
|
product: ${product_for_papertrail}
|
|
version: ${release_version}
|
|
filenames:
|
|
- "src/mongo_crypt_shared_v1-${push_name}-${push_arch}*"
|
|
|
|
################################################
|
|
# Task Groups #
|
|
################################################
|
|
task_groups:
|
|
- <<: *compile_task_group_template
|
|
name: archive_dist_test_TG
|
|
tasks:
|
|
- archive_dist_test
|
|
|
|
- <<: *compile_bazel_task_group_template
|
|
name: compile_bazel_TG
|
|
tasks:
|
|
- compile_dist_test_quick
|
|
|
|
- <<: *compile_bazel_task_group_template
|
|
name: run_bazel_TG
|
|
tasks:
|
|
- run_bazel_program
|
|
- bazel_archive_and_run_dist_test
|
|
|
|
# We want to run these sequentially to put less load on the cluster
|
|
- <<: *compile_task_group_template
|
|
name: hydrate_bazel_profile_TG
|
|
max_hosts: -1
|
|
tasks:
|
|
- hydrate_bazel_profile_fastbuild
|
|
- hydrate_required_bazel_profile_opt
|
|
- hydrate_bazel_profile_dbg
|
|
- hydrate_bazel_profile_dbg_aubsan
|
|
- hydrate_bazel_profile_dbg_tsan
|
|
|
|
# TODO(SERVER-99213): Remove when CI is using fission by default
|
|
# Run this in required to prevent breaks when building with fission on
|
|
- <<: *compile_task_group_template
|
|
name: hydrate_required_bazel_profile_TG
|
|
max_hosts: -1
|
|
tasks:
|
|
- hydrate_required_bazel_profile_opt
|
|
|
|
- <<: *compile_bazel_task_group_template
|
|
name: compile_bazel_windows_TG
|
|
tasks:
|
|
- run_bazel_program_windows
|
|
|
|
- <<: *compile_bazel_task_group_template
|
|
name: compile_bazel_macos_TG
|
|
tasks:
|
|
- compile_dist_test_quick
|
|
|
|
- <<: *compile_task_group_template
|
|
name: compile_upload_benchmarks_TG
|
|
tasks:
|
|
- compile_upload_benchmarks
|
|
# TODO(SERVER-86426): Figure out a workable long-term solution to benchmark debug symbols
|
|
# - compile_upload_benchmarks_debug
|
|
|
|
- <<: *compile_task_group_template
|
|
name: compile_and_archive_dist_test_TG
|
|
tasks:
|
|
- archive_dist_test
|
|
- archive_dist_test_debug
|
|
|
|
- <<: *compile_task_group_template
|
|
name: compile_and_archive_dist_test_then_package_TG
|
|
tasks:
|
|
- archive_dist_test
|
|
- archive_dist_test_debug
|
|
- package
|
|
|
|
- <<: *compile_task_group_template
|
|
name: compile_run_and_archive_dbtest_TG
|
|
tasks:
|
|
- compile_dbtest
|
|
- run_dbtest
|
|
- archive_dbtest
|
|
|
|
- <<: *compile_task_group_template
|
|
name: compile_test_serial_TG
|
|
tasks:
|
|
- archive_dist_test
|
|
- archive_dist_test_debug
|
|
- compile_dbtest
|
|
- run_dbtest
|
|
- archive_dbtest
|
|
|
|
- <<: *compile_task_group_template
|
|
name: compile_jstestshell_TG
|
|
tasks:
|
|
- archive_jstestshell
|
|
- archive_jstestshell_debug
|
|
|
|
- <<: *compile_task_group_template
|
|
name: compile_test_serial_future_git_tag_multiversion_TG
|
|
tasks:
|
|
- archive_dist_test_future_git_tag_multiversion
|
|
- archive_dist_test_debug_future_git_tag_multiversion
|
|
- compile_unittests_future_git_tag_multiversion
|
|
- run_unittests_future_git_tag_multiversion
|
|
- compile_dbtest_future_git_tag_multiversion
|
|
- run_dbtest_future_git_tag_multiversion
|
|
- archive_dbtest_future_git_tag_multiversion
|
|
- compile_all_future_git_tag_multiversion
|
|
|
|
- <<: *compile_task_group_template
|
|
name: compile_test_and_package_serial_TG
|
|
tasks:
|
|
- archive_dist_test
|
|
- archive_dist_test_debug
|
|
# package task here to reduce peak diskspace SERVER-86662
|
|
- package
|
|
- compile_dbtest
|
|
- run_dbtest
|
|
- archive_dbtest
|
|
|
|
# The *no_unittests_TG taskgroups are useful for static builds,
|
|
# were the static binaries of the unittests can take up to
|
|
# a terabyte of data.
|
|
- <<: *compile_task_group_template
|
|
name: compile_test_serial_no_unittests_TG
|
|
tasks:
|
|
- archive_dist_test
|
|
- archive_dist_test_debug
|
|
- compile_dbtest
|
|
- run_dbtest
|
|
- archive_dbtest
|
|
- compile_all_but_not_unittests
|
|
|
|
- <<: *compile_task_group_template
|
|
name: compile_test_and_package_serial_no_unittests_TG
|
|
tasks:
|
|
- archive_dist_test
|
|
- archive_dist_test_debug
|
|
# package task here to reduce peak diskspace SERVER-86662
|
|
- package
|
|
- compile_dbtest
|
|
- run_dbtest
|
|
- archive_dbtest
|
|
- compile_all_but_not_unittests
|
|
|
|
# TODO: SERVER-79886 Fix commented out tasks
|
|
- <<: *compile_task_group_template
|
|
name: compile_test_and_package_serial_lto_no_unittests_TG
|
|
tasks:
|
|
- archive_dist_test
|
|
- archive_dist_test_debug
|
|
# package task here to reduce peak diskspace SERVER-86662
|
|
- package
|
|
- compile_dbtest
|
|
# - run_dbtest
|
|
# - archive_dbtest
|
|
# - compile_all_but_not_unittests
|
|
|
|
# The *no_unittests_TG taskgroups are useful for static builds,
|
|
# were the static binaries of the unittests can take up to
|
|
# a terabyte of data.
|
|
- <<: *compile_task_group_template
|
|
name: compile_test_benchmark_and_package_serial_no_unittests_TG
|
|
tasks:
|
|
- archive_dist_test
|
|
- archive_dist_test_debug
|
|
# package task here to reduce peak diskspace SERVER-86662
|
|
- package
|
|
- compile_dbtest
|
|
- run_dbtest
|
|
- archive_dbtest
|
|
- compile_upload_benchmarks
|
|
# TODO(SERVER-86426): Figure out a workable long-term solution to benchmark debug symbols
|
|
# - compile_upload_benchmarks_debug
|
|
- compile_all_but_not_unittests
|
|
|
|
- <<: *compile_task_group_template
|
|
name: run_unit_tests_TG
|
|
max_hosts: -1
|
|
tasks:
|
|
- compile_libraries_for_unittests
|
|
- unit_test_group1
|
|
- unit_test_group2
|
|
- unit_test_group3
|
|
- unit_test_group4
|
|
- unit_test_group5
|
|
- unit_test_group6
|
|
- unit_test_group7
|
|
- unit_test_group8
|
|
|
|
- <<: *compile_task_group_template
|
|
name: run_unit_tests_no_sandbox_TG
|
|
max_hosts: -1
|
|
tasks:
|
|
- compile_libraries_for_unittests
|
|
- unit_test_group1_no_sandbox
|
|
- unit_test_group2_no_sandbox
|
|
- unit_test_group3_no_sandbox
|
|
- unit_test_group4_no_sandbox
|
|
- unit_test_group5_no_sandbox
|
|
- unit_test_group6_no_sandbox
|
|
- unit_test_group7_no_sandbox
|
|
- unit_test_group8_no_sandbox
|
|
|
|
- <<: *compile_task_group_template
|
|
name: compile_unittest_libaries_TG
|
|
max_hosts: -1
|
|
tasks:
|
|
- compile_libraries_for_unittests
|
|
|
|
# These `parallel` task groups are only appropriate for builders that
|
|
# use --linkstatic=False. Such builders are able to share all build
|
|
# artifacts, and therefore will not repeatedly re-link the same
|
|
# code. In that mode, it makes sense to run all of these tasks
|
|
# concurrently, since they will share state across machines and can
|
|
# complete faster than running them serially. We keep them in task
|
|
# groups so that if they do run on the same machine, they can avoid the
|
|
# cost of re-running the setup tasks.
|
|
- <<: *compile_task_group_template
|
|
name: compile_test_parallel_core_stream_TG
|
|
tasks:
|
|
- archive_dist_test
|
|
- archive_dist_test_debug
|
|
- compile_all_but_not_unittests
|
|
|
|
- <<: *compile_task_group_template
|
|
name: compile_test_parallel_core_stream_and_pretty_printer_tests_TG
|
|
tasks:
|
|
- archive_dist_test
|
|
- archive_dist_test_debug
|
|
- run_pretty_printer_tests
|
|
|
|
- <<: *compile_task_group_template
|
|
name: compile_all_but_not_unittests_TG
|
|
tasks:
|
|
- compile_all_but_not_unittests
|
|
|
|
- <<: *compile_task_group_template
|
|
name: compile_test_parallel_dbtest_stream_TG
|
|
tasks:
|
|
- compile_dbtest
|
|
- run_dbtest
|
|
- archive_dbtest
|
|
|
|
- name: crypt_build_debug_and_test
|
|
setup_task:
|
|
- func: "f_expansions_write"
|
|
- func: "set task expansion macros"
|
|
- func: "f_expansions_write"
|
|
- func: "get engflow creds"
|
|
setup_group_can_fail_task: true
|
|
setup_group:
|
|
- command: manifest.load
|
|
- func: "git get project and add git tag"
|
|
- func: "set task expansion macros"
|
|
- func: "f_expansions_write"
|
|
- func: "kill processes"
|
|
- func: "cleanup environment"
|
|
- func: "set up venv"
|
|
- func: "upload pip requirements"
|
|
- func: "get buildnumber"
|
|
max_hosts: 1
|
|
tasks:
|
|
- "crypt_create_debug_lib"
|
|
- "crypt_install_tests"
|
|
- "crypt_run_tests"
|