mirror of https://github.com/mongodb/mongo
553 lines
19 KiB
YAML
553 lines
19 KiB
YAML
variables:
|
|
# THIS HAS COPIES IN
|
|
# - etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml
|
|
# - etc/evergreen_yml_components/tasks/compile_tasks.yml
|
|
# - etc/evergreen_yml_components/tasks/compile_tasks_shared.yml
|
|
# ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES
|
|
- &compile_task_group_template
|
|
name: compile_task_group_template
|
|
max_hosts: 1
|
|
tasks: []
|
|
setup_task:
|
|
- func: "f_expansions_write"
|
|
- func: "set task expansion macros"
|
|
- func: "f_expansions_write"
|
|
- func: "override task timeout"
|
|
- func: "get engflow creds"
|
|
teardown_task:
|
|
- func: "f_expansions_write"
|
|
- func: "create bazel test report"
|
|
- func: "attach report"
|
|
- func: "attach artifacts"
|
|
- func: "attach local resmoke invocation"
|
|
- func: "attach multiversion download links"
|
|
- func: "kill processes"
|
|
- func: "save code coverage data"
|
|
- func: "save mongo coredumps"
|
|
- func: "generate hang analyzer tasks"
|
|
- func: "save failed unittests"
|
|
- func: "save bazel headers"
|
|
- func: "save bazel jvm dump"
|
|
- func: "save bazel exec logs"
|
|
- func: "save hang analyzer debugger files"
|
|
- func: "save disk statistics"
|
|
- func: "save system resource information"
|
|
- func: "save libfuzzertest corpora"
|
|
- func: "remove files"
|
|
vars:
|
|
files: >-
|
|
src/resmoke_error_code
|
|
src/*.gcda.gcov
|
|
src/gcov-intermediate-files.tgz
|
|
src/*.core src/*.mdmp src/*.core.gz src/*.mdmp.gz
|
|
mongo-coredumps.json
|
|
src/dist-unittests/bin/*
|
|
src/dist-unittests/lib/*
|
|
mongo-unittests.tgz
|
|
src/debugger*.*
|
|
src/mongo-hanganalyzer.tgz
|
|
diskstats.tgz
|
|
system-resource-info.tgz
|
|
${report_file|src/report.json}
|
|
${archive_file|src/archive.json}
|
|
setup_group_can_fail_task: true
|
|
setup_group:
|
|
- command: manifest.load
|
|
- func: "git get project and add git tag"
|
|
- func: "set task expansion macros"
|
|
- func: "f_expansions_write"
|
|
- func: "kill processes"
|
|
- func: "cleanup environment"
|
|
# The python virtual environment is installed in ${workdir}, which is created in
|
|
# "set up venv".
|
|
- func: "set up venv"
|
|
- func: "upload pip requirements"
|
|
- func: "f_expansions_write"
|
|
- func: "configure evergreen api credentials"
|
|
- func: "get buildnumber"
|
|
- func: "f_expansions_write"
|
|
- func: "set up credentials"
|
|
- func: "f_expansions_write"
|
|
- func: "use WiredTiger develop" # noop if ${use_wt_develop} is not "true"
|
|
- func: "f_expansions_write"
|
|
teardown_group:
|
|
- func: "f_expansions_write"
|
|
- func: "cleanup environment"
|
|
timeout:
|
|
- func: "f_expansions_write"
|
|
- func: "run hang analyzer"
|
|
- func: "wait for resmoke to shutdown"
|
|
- func: "save bazel run logs"
|
|
|
|
task_groups:
|
|
- <<: *compile_task_group_template
|
|
name: compile_and_package_serial_no_unittests_TG
|
|
tasks:
|
|
- archive_dist_test
|
|
- archive_dist_test_debug
|
|
|
|
tasks:
|
|
- &version_expansions_gen
|
|
name: version_expansions_gen
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
priority: 10
|
|
commands:
|
|
- command: manifest.load
|
|
- func: "git get shallow project"
|
|
- func: "f_expansions_write"
|
|
- func: "restore git history and tags"
|
|
- func: "add git tag"
|
|
- func: "kill processes"
|
|
- func: "cleanup environment"
|
|
- func: "set up venv"
|
|
- func: "upload pip requirements"
|
|
- func: "generate and upload version expansions"
|
|
- func: "generate and upload resmoke constants"
|
|
|
|
- <<: *version_expansions_gen
|
|
name: version_expansions_future_git_tag_multiversion_gen
|
|
|
|
# Sys-perf relies on the name of this task, please reach out before changing it.
|
|
- &archive_dist_test
|
|
name: archive_dist_test
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "activate task"
|
|
vars:
|
|
task_to_activate: ${archive_dist_test_debug_task_name|archive_dist_test_debug}
|
|
skip_for_patch_author: sys-perf-user
|
|
skip_activate_task: ${skip_archive_dist_test_debug_activate|False}
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: >-
|
|
archive-dist-test-stripped
|
|
${additional_compile_targets|}
|
|
bazel_args: >-
|
|
--config=evg
|
|
--config=opt_profiled
|
|
- func: "f_expansions_write"
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
add_expansions_to_env: true
|
|
args:
|
|
- "src/evergreen/run_python_script.sh"
|
|
- "evergreen/macos_notary.py"
|
|
- "bazel-bin/dist-test-stripped.${ext|tgz}"
|
|
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
add_expansions_to_env: true
|
|
args:
|
|
- "src/evergreen/run_python_script.sh"
|
|
- "buildscripts/s3_binary/sha256sum.py"
|
|
- "bazel-bin/dist-test-stripped.${ext|tgz}"
|
|
- command: s3.put
|
|
params:
|
|
optional: true
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/bazel-bin/dist-test-stripped.${ext|tgz}
|
|
remote_file: ${mongo_binaries}
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: application/gzip
|
|
# Sys-perf relies on this display name, please reach out before changing it.
|
|
display_name: Binaries
|
|
- command: s3.put
|
|
params:
|
|
optional: true
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/bazel-bin/dist-test-stripped.${ext|tgz}.sha256
|
|
remote_file: ${mongo_binaries}.sha256
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: text/plain
|
|
display_name: Binaries SHA256
|
|
|
|
- func: "f_expansions_write"
|
|
- func: "gen feature flags"
|
|
- func: "fetch resmoke constants"
|
|
- func: "bazel run"
|
|
vars:
|
|
target: >-
|
|
//buildscripts:archive_artifacts --
|
|
--output=artifacts.tgz
|
|
--exclude="*_test.pdb"
|
|
--base_dir=".."
|
|
"./**/*.gcno"
|
|
"src/.bazel*"
|
|
"src/.npmrc"
|
|
"src/.resmoke_mongo_release_values.yml"
|
|
"src/.resmoke_mongo_version.yml"
|
|
"src/BUILD.bazel"
|
|
"src/MODULE.bazel"
|
|
"src/WORKSPACE.bazel"
|
|
"src/bazel/**"
|
|
"src/buildscripts/**"
|
|
"src/docker_compose/**"
|
|
"src/etc/*san.suppressions"
|
|
"src/etc/backports_required_for_multiversion_tests.yml"
|
|
"src/etc/evergreen.yml"
|
|
"src/etc/evergreen_nightly.yml"
|
|
"src/etc/evergreen_timeouts.yml"
|
|
"src/etc/evergreen_yml_components/**"
|
|
"src/etc/expansions.default.yml"
|
|
"src/etc/macos_dev_entitlements.xml"
|
|
"src/etc/repo_config.yaml"
|
|
"src/evergreen/**"
|
|
"src/jsconfig.json"
|
|
"src/jstests/**"
|
|
"src/package.json"
|
|
"src/patch_files.txt"
|
|
"src/patch_test_tags.tgz"
|
|
"src/pnpm-lock.yaml"
|
|
"src/poetry.lock"
|
|
"src/poetry_requirements.txt"
|
|
"src/pyproject.toml"
|
|
"src/src/**/*.idl"
|
|
"src/src/**/*.yml"
|
|
"src/src/mongo/client/sdam/json_tests/sdam_tests/**"
|
|
"src/src/mongo/client/sdam/json_tests/server_selection_tests/**"
|
|
"src/src/mongo/db/modules/atlas/evergreen/**"
|
|
"src/src/mongo/db/modules/atlas/jstests/**"
|
|
"src/src/mongo/db/modules/enterprise/docs/**"
|
|
"src/src/mongo/db/modules/enterprise/jstests/**"
|
|
"src/src/mongo/db/modules/enterprise/src/streams/build/*"
|
|
"src/src/mongo/db/modules/enterprise/src/streams/aspio/**"
|
|
"src/src/mongo/db/modules/subscription/jstests/**"
|
|
"src/src/mongo/db/query/benchmark/data_generator/**"
|
|
"src/src/mongo/db/query/query_tester/tests/**"
|
|
"src/src/mongo/util/options_parser/test_config_files/**"
|
|
"src/src/mongo/util/version/**"
|
|
"src/src/third_party/JSON-Schema-Test-Suite/tests/draft4/**"
|
|
"src/src/third_party/abseil-cpp/**"
|
|
"src/src/third_party/grpc/**"
|
|
"src/src/third_party/mock_ocsp_responder/**"
|
|
"src/src/third_party/protobuf/**"
|
|
"src/src/third_party/schemastore.org/**"
|
|
"src/tools/**"
|
|
"src/x509/**"
|
|
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: artifacts.tgz
|
|
remote_file: ${mongo_artifacts}
|
|
bucket: mciuploads
|
|
permissions: private
|
|
visibility: signed
|
|
content_type: application/tar
|
|
display_name: Artifacts
|
|
|
|
- command: archive.targz_pack
|
|
params:
|
|
target: "venv.tgz"
|
|
source_dir: "./"
|
|
include:
|
|
- "./venv/**"
|
|
- "./venv_readme.txt"
|
|
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: venv.tgz
|
|
remote_file: ${mongo_venv}
|
|
bucket: mciuploads
|
|
permissions: private
|
|
visibility: signed
|
|
content_type: application/tar
|
|
display_name: Python venv (see included venv_readme.txt)
|
|
|
|
- <<: *archive_dist_test
|
|
name: archive_dist_test_future_git_tag_multiversion
|
|
depends_on:
|
|
- name: version_expansions_future_git_tag_multiversion_gen
|
|
|
|
- &archive_dist_test_debug
|
|
name: archive_dist_test_debug
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
depends_on:
|
|
- name: archive_dist_test
|
|
commands:
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: >-
|
|
archive-dist-test-debug install-dist-test
|
|
bazel_args: >-
|
|
--config=evg
|
|
--config=opt_profiled
|
|
- func: "upload debugsymbols"
|
|
- func: "f_expansions_write"
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
args:
|
|
- "./src/evergreen/record_mongodb_server_version.sh"
|
|
- "./bazel-bin/install/bin/mongod"
|
|
- "./version_info_mongod.txt"
|
|
- command: s3.put
|
|
params:
|
|
optional: true
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/version_info_mongod.txt
|
|
remote_file: ${project}/${build_variant}/${version_id}/version-info-mongod-${task_id}.txt
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: text/plain
|
|
display_name: mongod --version output
|
|
|
|
- <<: *archive_dist_test_debug
|
|
name: archive_dist_test_debug_future_git_tag_multiversion
|
|
depends_on:
|
|
- name: archive_dist_test_future_git_tag_multiversion
|
|
|
|
- name: crypt_create_lib
|
|
tags:
|
|
[
|
|
"assigned_to_jira_team_server_security",
|
|
"release_critical",
|
|
"incompatible_community",
|
|
"incompatible_tsan",
|
|
"incompatible_aubsan",
|
|
"requires_compile_variant",
|
|
"requires_large_host",
|
|
"crypt",
|
|
"bazel_check",
|
|
]
|
|
depends_on:
|
|
- name: version_expansions_gen
|
|
variant: generate-tasks-for-version
|
|
commands:
|
|
- func: "do bazel setup"
|
|
- func: "f_expansions_write"
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: archive-mongo_crypt-stripped
|
|
bazel_args: >-
|
|
--config=evg_crypt
|
|
task_compile_flags: --linkstatic=True ${crypt_task_compile_flags}
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
args:
|
|
- "src/evergreen/crypt_run_tests.sh"
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
add_expansions_to_env: true
|
|
args:
|
|
- "src/evergreen/run_python_script.sh"
|
|
- "evergreen/macos_notary.py"
|
|
- "bazel-bin/mongo_crypt-stripped.${ext|tgz}"
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: "src/bazel-bin/mongo_crypt-stripped.${ext|tgz}"
|
|
remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shared_v1-${version}.${ext|tgz}"
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: ${content_type|application/tar}
|
|
display_name: "Mongo crypt Library"
|
|
|
|
- name: crypt_lib_package
|
|
tags:
|
|
[
|
|
"assigned_to_jira_team_devprod_build",
|
|
"crypt",
|
|
"incompatible_aubsan",
|
|
"incompatible_community",
|
|
"incompatible_mac",
|
|
"incompatible_tsan",
|
|
"incompatible_windows",
|
|
"release_critical",
|
|
"requires_compile_variant",
|
|
"requires_large_host",
|
|
]
|
|
depends_on:
|
|
- name: crypt_create_lib
|
|
commands:
|
|
- func: "do non-compile setup"
|
|
- func: "get and apply version expansions"
|
|
- func: "f_expansions_write"
|
|
- command: s3.get
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongo_crypt_shared_v1-${version}.${ext|tgz}
|
|
remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shared_v1-${version}.${ext|tgz}"
|
|
bucket: mciuploads
|
|
- func: "run packager_crypt.py"
|
|
- command: archive.targz_pack
|
|
params:
|
|
target: "packages.tgz"
|
|
source_dir: "src"
|
|
include:
|
|
- "repo/**"
|
|
- command: s3.put
|
|
params:
|
|
optional: true
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: packages.tgz
|
|
remote_file: ${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: application/tar
|
|
display_name: Packages
|
|
|
|
- name: package
|
|
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
|
|
depends_on:
|
|
- name: archive_dist_test
|
|
commands:
|
|
- func: "f_expansions_write"
|
|
- func: "bazel compile"
|
|
vars:
|
|
targets: >-
|
|
archive-dist-stripped
|
|
archive-dist-debug
|
|
archive-mongo-stripped
|
|
archive-mongo-debug
|
|
${additional_package_targets|}
|
|
bazel_args: >-
|
|
--config=evg
|
|
--config=opt_profiled
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
add_expansions_to_env: true
|
|
args:
|
|
- "src/evergreen/run_python_script.sh"
|
|
- "evergreen/macos_notary.py"
|
|
- "bazel-bin/dist-stripped.${ext|tgz}"
|
|
- func: "f_expansions_write"
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
args:
|
|
- "src/evergreen/package.sh"
|
|
- command: s3.put
|
|
params:
|
|
optional: true
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/bazel-bin/dist-stripped.${ext|tgz}
|
|
remote_file: ${project}/${build_variant}/${revision}/dist/mongo-${build_id}.${ext|tgz}
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: application/tar
|
|
display_name: Dist Tarball
|
|
- command: s3.put
|
|
params:
|
|
optional: true
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/bazel-bin/dist-debug.${ext|tgz}
|
|
remote_file: ${project}/${build_variant}/${revision}/dist/mongo-${build_id}-debugsymbols.${ext|tgz}
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: application/tar
|
|
display_name: Dist Debugsymbols
|
|
- command: subprocess.exec
|
|
params:
|
|
binary: bash
|
|
add_expansions_to_env: true
|
|
args:
|
|
- "src/evergreen/run_python_script.sh"
|
|
- "buildscripts/s3_binary/sha256sum.py"
|
|
- "bazel-bin/mongo-stripped.${ext|tgz}"
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/bazel-bin/mongo-stripped.${ext|tgz}
|
|
remote_file: ${mongo_jstestshell}
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: ${content_type|application/gzip}
|
|
display_name: Jstestshell
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/bazel-bin/mongo-stripped.${ext|tgz}.sha256
|
|
remote_file: ${mongo_jstestshell}.sha256
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: text/plain
|
|
display_name: Jstestshell SHA256
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/bazel-bin/mongo-debug.${ext|tgz}
|
|
remote_file: ${mongo_jstestshell_debugsymbols}
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: ${content_type|application/gzip}
|
|
display_name: Jstestshell Debugsymbols
|
|
- command: s3.put
|
|
params:
|
|
optional: true
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/bazel-bin/mongocryptd-stripped.${ext|tgz}
|
|
remote_file: ${mongo_cryptd}
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: ${content_type|application/gzip}
|
|
display_name: CryptD Binaries
|
|
- command: s3.put
|
|
params:
|
|
optional: true
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/bazel-bin/mongocryptd-debug.${ext|tgz}
|
|
remote_file: ${mongo_cryptd_debugsymbols}
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: ${content_type|application/gzip}
|
|
display_name: CryptD Debugsymbols
|
|
- func: "run packager.py"
|
|
- command: archive.targz_pack
|
|
params:
|
|
target: "packages.tgz"
|
|
source_dir: "src"
|
|
include:
|
|
- "repo/**"
|
|
- "bazel-bin/src/mongo/installer/**.msi"
|
|
- command: s3.put
|
|
params:
|
|
optional: true
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: packages.tgz
|
|
remote_file: ${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: application/tar
|
|
display_name: Packages
|
|
# We perform package task generally in the middle of larger task groups running
|
|
# on the same host serially. In order to reduce peak required disk space, we can delete
|
|
# these generated files since we have already uploaded them to aws at this point.
|
|
- func: "remove files"
|
|
vars:
|
|
files: >-
|
|
packages.tgz
|
|
src/bazel-bin/mongocryptd-debug.${ext|tgz}
|
|
src/bazel-bin/mongocryptd-stripped.${ext|tgz}
|
|
src/bazel-bin/mongo-debug.${ext|tgz}
|
|
src/bazel-bin/mongo-stripped.${ext|tgz}
|
|
src/bazel-bin/dist-debug.${ext|tgz}
|
|
src/bazel-bin/dist-stripped.${ext|tgz}
|