mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-93965 Error on evergreen project config validation (#28789) 

GitOrigin-RevId: d44642c997d4982fdb200779a47d9f1d65bc40a6
This commit is contained in:
Mikhail Shchatko 2024-11-07 12:12:57 +02:00 committed by MongoDB Bot
parent 5af7bc78e1
commit d433e2b9b0
12 changed files with 193 additions and 224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/CODEOWNERS vendored
View File

@ -86,7 +86,6 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
/docs/**/golden_data_test_framework.md @10gen/query-optimization @svc-auto-approve-bot
# The following patterns are parsed from ./etc/OWNERS.yml
/etc/**/yamllint_config.yml @10gen/devprod-services-integrations @svc-auto-approve-bot
/etc/**/third_party_components.yml @10gen/server-security @svc-auto-approve-bot
/etc/**/system_perf.yml @10gen/devprod-performance-analysis @10gen/devprod-performance-infrastructure @10gen/performance @svc-auto-approve-bot

6
.gitignore vendored
View File

@ -307,3 +307,9 @@ buildscripts/antithesis/base_images/workload/tsan.suppressions
# default download location for resmoke core-analyzer
core-analyzer
# interim Evergreen yml files for linting
etc/evaluated_evergreen.yml
etc/evaluated_evergreen_nightly.yml
etc/evaluated_system_perf.yml
etc/trimmed_system_perf.yml

79
buildscripts/ciconfig/evergreen.py
View File

@ -23,43 +23,54 @@ ASAN_SIGNATURE = "detect_leaks=1"
LOGGER = structlog.get_logger(__name__)
def find_evergreen_binary(evergreen_binary):
"""Find evergreen binary."""
if not evergreen_binary:
return
if shutil.which(evergreen_binary):
return shutil.which(evergreen_binary)
# On windows in python3.8 there was an update to no longer use HOME in os.path.expanduser
# However, cygwin is weird and has HOME but not USERPROFILE
# So we just check if HOME is set and USERPROFILE is not
# Then we just set USERPROFILE and unset it after
# Bug is here: https://bugs.python.org/issue36264
prev_environ = os.environ.copy()
if sys.platform in ("win32", "cygwin"):
LOGGER.info(f"Previous os.environ={os.environ} before updating 'USERPROFILE'")
if "HOME" in os.environ:
os.environ["USERPROFILE"] = os.environ["HOME"]
else:
LOGGER.warn(
"'HOME' environment variable unset."
" This will likely cause us to be unable to find evergreen binary."
)
default_evergreen_location = os.path.expanduser(os.path.join("~", "evergreen"))
# Restore environment if it was modified above on windows
os.environ.clear()
os.environ.update(prev_environ)
if os.path.exists(default_evergreen_location):
evergreen_binary = default_evergreen_location
elif os.path.exists(f"{default_evergreen_location}.exe"):
evergreen_binary = f"{default_evergreen_location}.exe"
else:
raise EnvironmentError(
f"Executable {evergreen_binary} (default location: {default_evergreen_location})"
f" does not exist or is not in the PATH. PATH={os.environ.get('PATH')}"
)
return evergreen_binary
def parse_evergreen_file(path, evergreen_binary="evergreen"):
"""Read an Evergreen file and return EvergreenProjectConfig instance."""
evergreen_binary = find_evergreen_binary(evergreen_binary)
if evergreen_binary:
if not shutil.which(evergreen_binary):
# On windows in python3.8 there was an update to no longer use HOME in os.path.expanduser
# However, cygwin is weird and has HOME but not USERPROFILE
# So we just check if HOME is set and USERPROFILE is not
# Then we just set USERPROFILE and unset it after
# Bug is here: https://bugs.python.org/issue36264
prev_environ = os.environ.copy()
if sys.platform in ("win32", "cygwin"):
LOGGER.info(f"Previous os.environ={os.environ} before updating 'USERPROFILE'")
if "HOME" in os.environ:
os.environ["USERPROFILE"] = os.environ["HOME"]
else:
LOGGER.warn(
"'HOME' enviorment variable unset. This will likely cause us to be unable to find evergreen binary."
)
default_evergreen_location = os.path.expanduser(os.path.join("~", "evergreen"))
# Restore enviorment if it was modified above on windows
os.environ.clear()
os.environ.update(prev_environ)
if os.path.exists(default_evergreen_location):
evergreen_binary = default_evergreen_location
elif os.path.exists(f"{default_evergreen_location}.exe"):
evergreen_binary = f"{default_evergreen_location}.exe"
else:
raise EnvironmentError(
f"Executable {evergreen_binary} (default location: {default_evergreen_location}) does not exist or is not in the PATH. PATH={os.environ.get('PATH')}"
)
else:
evergreen_binary = shutil.which(evergreen_binary)
# Call 'evergreen evaluate path' to pre-process the project configuration file.
cmd = [evergreen_binary, "evaluate", path]
result = subprocess.run(cmd, capture_output=True, text=True)

117
buildscripts/validate_evg_project_config.py Normal file
View File

@ -0,0 +1,117 @@
import os.path
import re
import subprocess
import sys
from collections import defaultdict
import structlog
import typer
from typing_extensions import Annotated
from buildscripts.ciconfig.evergreen import find_evergreen_binary
LOGGER = structlog.get_logger(__name__)
DEFAULT_LOCAL_EVG_AUTH_CONFIG = os.path.expanduser("~/.evergreen.yml")
DEFAULT_EVG_PROJECT_NAME = "mongodb-mongo-master"
DEFAULT_EVG_NIGHTLY_PROJECT_NAME = "mongodb-mongo-master-nightly"
DEFAULT_EVG_PROJECT_CONFIG = "etc/evergreen.yml"
DEFAULT_EVG_NIGHTLY_PROJECT_CONFIG = "etc/evergreen_nightly.yml"
ALLOWABLE_EVG_VALIDATE_MESSAGE_REGEXES = [
re.compile(r".*buildvariant .+ has unmatched selector: .+"),
re.compile(r".*buildvariant .+ has unmatched criteria: .+"),
]
ALLOWABLE_IF_NOT_IN_ALL_PROJECTS_EVG_VALIDATE_MESSAGE_REGEXES = [
re.compile(r".*task .+ defined but not used by any variants; consider using or disabling.*"),
]
HORIZONTAL_LINE = "-" * 100
def main(
evg_project_name: Annotated[
str, typer.Option(help="Evergreen project name")
] = DEFAULT_EVG_PROJECT_NAME,
evg_auth_config: Annotated[
str, typer.Option(help="Evergreen auth config file")
] = DEFAULT_LOCAL_EVG_AUTH_CONFIG,
):
evg_project_config_map = {evg_project_name: DEFAULT_EVG_NIGHTLY_PROJECT_CONFIG}
if evg_project_name == DEFAULT_EVG_PROJECT_NAME:
evg_project_config_map = {
DEFAULT_EVG_PROJECT_NAME: DEFAULT_EVG_PROJECT_CONFIG,
DEFAULT_EVG_NIGHTLY_PROJECT_NAME: DEFAULT_EVG_NIGHTLY_PROJECT_CONFIG,
}
shared_evg_validate_messages = []
error_on_evg_validate_messages = defaultdict(list)
num_of_projects = len(evg_project_config_map)
evergreen_bin = find_evergreen_binary("evergreen")
for project, project_config in evg_project_config_map.items():
cmd = [
evergreen_bin,
"--config",
evg_auth_config,
"validate",
"--project",
project,
"--path",
project_config,
]
LOGGER.info(f"Running command: {cmd}")
result = subprocess.run(cmd, capture_output=True, text=True)
interesting_messages = result.stdout.strip().split("\n")[:-1]
for message in interesting_messages:
if num_of_projects > 1 and any(
regex.match(message)
for regex in ALLOWABLE_IF_NOT_IN_ALL_PROJECTS_EVG_VALIDATE_MESSAGE_REGEXES
):
shared_evg_validate_messages.append(message)
continue
if any(regex.match(message) for regex in ALLOWABLE_EVG_VALIDATE_MESSAGE_REGEXES):
continue
error_on_evg_validate_messages[project].append(message)
error_on_shared_evg_validate_messages = []
for message in set(shared_evg_validate_messages):
if shared_evg_validate_messages.count(message) == num_of_projects:
error_on_shared_evg_validate_messages.append(message)
exit_code = 0
all_configs = list(evg_project_config_map.values())
all_projects = list(evg_project_config_map.keys())
for project, errors in error_on_evg_validate_messages.items():
if len(errors) > 0:
exit_code = 1
project_config = evg_project_config_map[project]
LOGGER.info(HORIZONTAL_LINE)
LOGGER.error(f"Config '{project_config}' for '{project}' evergreen project has errors:")
for error in errors:
LOGGER.error(error)
if len(error_on_shared_evg_validate_messages) > 0:
exit_code = 1
LOGGER.info(HORIZONTAL_LINE)
LOGGER.error(
f"Configs {all_configs} for evergreen projects {all_projects} have errors"
f" (they can be fixed in either config):"
)
for error in error_on_shared_evg_validate_messages:
LOGGER.error(error)
if exit_code == 0:
LOGGER.info(HORIZONTAL_LINE)
LOGGER.info(
f"Config(s) {all_configs} for evergreen project(s) {all_projects} is(are) valid"
)
sys.exit(exit_code)
if __name__ == "__main__":
typer.run(main)

3
etc/OWNERS.yml
View File

@ -1,8 +1,5 @@
version: 1.0.0
filters:
- "yamllint_config.yml":
approvers:
- 10gen/devprod-services-integrations
- "third_party_components.yml":
approvers:
- 10gen/server-security

4
etc/evergreen_yml_components/definitions.yml
View File

@ -100,7 +100,7 @@ functions:
- command: git.get_project
params:
directory: ${git_project_directory|src}
clone_depth: 1000
clone_depth: 1000
- *restore_git_history_and_tags
"add git tag": &add_git_tag
@ -707,7 +707,7 @@ functions:
"upload network diagnostics": &upload_network_diagnostics
command: s3.put
dipslay_name: "upload network diagnostics"
display_name: "upload network diagnostics"
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}

65
etc/evergreen_yml_components/tasks/compile_tasks.yml
View File

@ -356,6 +356,8 @@ tasks:
- name: compile_upload_benchmarks_debug
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
# TODO(SERVER-86426): Figure out a workable long-term solution to benchmark debug symbols
disable: true
depends_on:
- name: compile_upload_benchmarks
commands:
@ -667,22 +669,16 @@ tasks:
display_name: Libdeps Graph Data
## compile_all - build all scons targets ##
- &compile_all
name: compile_all
- name: compile_all_future_git_tag_multiversion
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
depends_on:
- name: compile_dist_test
- name: compile_dist_test_future_git_tag_multiversion
commands:
- func: "scons compile"
vars:
targets: install-all-meta
compiling_for_test: true
- <<: *compile_all
name: compile_all_future_git_tag_multiversion
depends_on:
- name: compile_dist_test_future_git_tag_multiversion
- name: compile_all_but_not_unittests
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
depends_on:
@ -738,11 +734,10 @@ tasks:
split: 4
## compile_unittests ##
- &compile_unittests
name: compile_unittests
- name: compile_unittests_future_git_tag_multiversion
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
depends_on:
- name: compile_dist_test
- name: compile_dist_test_future_git_tag_multiversion
commands:
- func: "scons compile"
vars:
@ -841,21 +836,11 @@ tasks:
install_dir: build/install/bin
exec_timeout_secs: 32400 # 9 hours
- <<: *compile_unittests
name: compile_unittests_future_git_tag_multiversion
depends_on:
- name: compile_dist_test_future_git_tag_multiversion
## A copy of the compile_unittests task for the recorded unittest taskgroup ##
- <<: *compile_unittests
name: compile_unittests_for_recorded_unittest
## run_unittests ##
- &run_unittests
name: run_unittests
- name: run_unittests_future_git_tag_multiversion
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
depends_on:
- name: compile_unittests
- name: compile_unittests_future_git_tag_multiversion
commands:
- func: "f_expansions_write"
- func: "run diskstats"
@ -867,40 +852,6 @@ tasks:
suite: unittests
install_dir: build/install/bin
- <<: *run_unittests
name: run_unittests_future_git_tag_multiversion
depends_on:
- name: compile_unittests_future_git_tag_multiversion
## run_unittests with UndoDB live-record ##
#- name: run_unittests_with_recording
# tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
# depends_on:
# - name: compile_unittests_for_recorded_unittest
# commands:
# - *f_expansions_write
# - func: "run diskstats"
# - func: "f_expansions_write"
# - func: "monitor process threads"
# - func: "collect system resource info"
# - command: subprocess.exec
# params:
# binary: bash
# args:
# - "./src/evergreen/undo_wiki_page.sh"
# - command: attach.artifacts
# params:
# files:
# - undo_wiki_page_location.json
# - func: "run tests"
# vars:
# suite: unittests
# record_with: --recordWith /opt/undodb5/bin/live-record
# # Start fewer jobs since there's a constant amount of overhead of starting
# # live-record for each job.
# resmoke_jobs_factor: 0.3
# install_dir: build/install/bin
##compile_and_archive_libfuzzertests - build libfuzzertests ##
- name: compile_and_archive_libfuzzertests
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]

94
etc/evergreen_yml_components/tasks/misc_tasks.yml
View File

@ -348,42 +348,6 @@ tasks:
resmoke_args: >-
--runAllFeatureFlagTests
- name: build_metrics_tasks_gen
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
depends_on:
- name: version_expansions_gen
variant: generate-tasks-for-version
commands:
- func: "do scons setup"
- func: "f_expansions_write"
- command: subprocess.exec
params:
binary: bash
add_expansions_to_env: true
args:
- "src/evergreen/run_python_script.sh"
- "buildscripts/evergreen_gen_build_metrics_tasks.py"
- command: archive.targz_pack
params:
target: build_metrics_task_gen.tgz
source_dir: "src"
include:
- "build_metrics_task_gen.json"
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: build_metrics_task_gen.tgz
remote_file: ${project}/${build_variant}/${revision}/build-metrics/${task_name}-${build_id}.tgz
bucket: mciuploads
permissions: public-read
content_type: application/gzip
display_name: Build Metrics Task Gen JSON
- command: generate.tasks
params:
files:
- src/build_metrics_task_gen.json
- name: check_feature_flag_tags
tags:
[
@ -502,36 +466,6 @@ tasks:
- "src/evergreen/run_python_script.sh"
- "buildscripts/iwyu/test/run_tests.py"
# Smoke test to ensure the Server still works with Jepsen
- <<: *run_jepsen_template
name: jepsen-config_fuzzer_smoke
tags: ["assigned_to_jira_team_server_repl", "experimental"]
commands:
- func: "do setup"
- func: "do jepsen setup"
- func: "setup jepsen config fuzzer"
- func: "run jepsen test"
vars:
<<: *jepsen_config_vars
mongod_conf: --mongod-conf mongod.conf
jepsen_storage_engine: --storage-engine wiredTiger
jepsen_test_name: read-concern-majority
jepsen_time_limit: --time-limit 120
# Smoke test to ensure the Server still works with Jepsen
- <<: *run_jepsen_template
name: jepsen-smoke
tags: ["assigned_to_jira_team_server_repl", "experimental"]
commands:
- func: "do setup"
- func: "do jepsen setup"
- func: "run jepsen test"
vars:
<<: *jepsen_config_vars
jepsen_storage_engine: --storage-engine wiredTiger
jepsen_test_name: read-concern-majority
jepsen_time_limit: --time-limit 120
- <<: *run_jepsen_template
name: jepsen_config_fuzzer_list-append
tags: ["assigned_to_jira_team_server_repl", "experimental", "jepsen_docker"]
@ -1122,6 +1056,17 @@ tasks:
binary: bash
args:
- "src/evergreen/lint_yaml.sh"
- func: "configure evergreen api credentials"
- command: subprocess.exec
type: test
params:
binary: bash
args:
- "src/evergreen/run_python_script_with_report.sh"
- "validate-evergreen-project-config-yaml"
- "buildscripts/validate_evg_project_config.py"
- "--evg-project-name=${project}"
- "--evg-auth-config=.evergreen.yml"
- name: bazel_coverage
tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
@ -1215,23 +1160,6 @@ tasks:
<<: *powercycle_remote_credentials
timeout_secs: 1800 # 30 minute timeout for no output
- name: powercycle_gen
tags:
[
"assigned_to_jira_team_server_storage_engines",
"experimental",
"incompatible_windows",
]
commands:
- func: "generate powercycle tasks"
vars:
task_names: >-
powercycle
powercycle_kill_mongod
powercycle_replication_smalloplog
powercycle_syncdelay
num_tasks: 1
- name: powercycle_kill_mongod
tags:
[

7
etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml
View File

@ -155,13 +155,6 @@ tasks:
commands:
- func: "generate resmoke tasks"
- <<: *gen_burn_in_task_template
name: burn_in_tasks_gen
tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
patch_only: true
commands:
- func: "generate resmoke tasks"
- <<: *gen_burn_in_task_template
name: burn_in_tests_gen
tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]

18
etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml
View File

@ -235,10 +235,11 @@ buildvariants:
- name: .crypt
- name: crypt_build_debug_and_test
- <<: *generic_linux_compile_params
name: &linux-x86-dynamic-grpc linux-x86-dynamic-grpc
- name: &linux-x86-dynamic-grpc linux-x86-dynamic-grpc
display_name: "~ Linux x86 Shared Library Enterprise with GRPC"
cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
run_on:
- rhel8.8-xlarge
stepback: false
expansions:
<<: *generic_linux_compile_expansions
@ -298,10 +299,6 @@ buildvariants:
# buildscripts/resmokeconfig/fully_disabled_feature_flags.yml
test_flags: >-
--runAllFeatureFlagTests
# Uncomment expansion and `burn_in_tasks_gen` task below and add resmoke task name to burn-in.
# WARNING! Task splitting is not supported for burn-in tasks. Large unsplitted `_gen` tasks may
# run too long and hit execution timeouts.
# burn_in_task_name: jsCore
depends_on:
- name: archive_dist_test
variant: *linux_x86_dynamic_compile_variant_name
@ -311,13 +308,6 @@ buildvariants:
# Without this we are adding extra dependencies on evergreen and it is causing strain
omit_generated_tasks: true
tasks:
# - name: burn_in_tasks_gen
# depends_on:
# - name: version_burn_in_gen
# variant: generate-tasks-for-version
# omit_generated_tasks: true
# - name: archive_dist_test
# variant: *linux_x86_dynamic_compile_variant_name
- name: .development_critical !.requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_all_feature_flags
- name: .development_critical .requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_all_feature_flags
distros:
@ -347,7 +337,6 @@ buildvariants:
run_on:
- rhel8.8-small
stepback: false
activate: true
expansions: &enterprise-rhel-8-64-bit-dynamic-embedded-router-expansions
<<: *enterprise-rhel-8-64-bit-dynamic-expansions
test_flags: >-
@ -368,7 +357,6 @@ buildvariants:
run_on:
- rhel8.8-small
stepback: false
activate: true
expansions:
&enterprise-rhel-8-64-bit-dynamic-replica-set-endpoint-expansions
<<: *enterprise-rhel-8-64-bit-dynamic-expansions

22
etc/evergreen_yml_components/variants/windows/test_dev.yml
View File

@ -109,18 +109,7 @@ buildvariants:
--runAllFeatureFlagTests
--excludeWithAnyTags=incompatible_with_windows_tls
external_auth_jobs_max: 1
# Uncomment expansion and `burn_in_tasks_gen` task below and add resmoke task name to burn-in.
# WARNING! Task splitting is not supported for burn-in tasks. Large unsplitted `_gen` tasks may
# run too long and hit execution timeouts.
# burn_in_task_name: jsCore
tasks:
# - name: burn_in_tasks_gen
# depends_on:
# - name: version_burn_in_gen
# variant: generate-tasks-for-version
# omit_generated_tasks: true
# - name: archive_dist_test
# variant: *windows_compile_variant_name
- name: .development_critical !.requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.incompatible_all_feature_flags .requires_execution_on_windows_patch_build
- name: .development_critical .requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.incompatible_all_feature_flags .requires_execution_on_windows_patch_build
distros:
@ -159,18 +148,7 @@ buildvariants:
--runAllFeatureFlagTests
--excludeWithAnyTags=incompatible_with_windows_tls
external_auth_jobs_max: 1
# Uncomment expansion and `burn_in_tasks_gen` task below and add resmoke task name to burn-in.
# WARNING! Task splitting is not supported for burn-in tasks. Large unsplitted `_gen` tasks may
# run too long and hit execution timeouts.
# burn_in_task_name: jsCore
tasks:
# - name: burn_in_tasks_gen
# depends_on:
# - name: version_burn_in_gen
# variant: generate-tasks-for-version
# omit_generated_tasks: true
# - name: archive_dist_test
# variant: *windows_compile_variant_name
- name: .development_critical !.requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.incompatible_all_feature_flags
- name: .development_critical .requires_large_host !.requires_compile_variant !.incompatible_development_variant !.incompatible_windows !.incompatible_all_feature_flags
distros:

1
etc/yamllint_config.yml
View File

@ -13,3 +13,4 @@ ignore:
- etc/evaluated_evergreen.yml
- etc/evaluated_evergreen_nightly.yml
- etc/evaluated_system_perf.yml
- etc/trimmed_system_perf.yml