From 7ccc14bf912324bb75f14c0a349edab8ad73aaca Mon Sep 17 00:00:00 2001 From: Zack Winter <3457246+zackwintermdb@users.noreply.github.com> Date: Tue, 15 Jul 2025 18:19:08 -0700 Subject: [PATCH] SERVER-101034 Use rules_lint shfmt formatter (#38448) GitOrigin-RevId: e8ef1ba2000e12fa2cd5a115a9ceeab92332e938 --- .editorconfig | 9 + .github/CODEOWNERS | 1 + OWNERS.yml | 3 + bazel/format/BUILD.bazel | 6 +- bazel/format/rules_lint_format_wrapper.py | 23 -- bazel/lint.sh | 4 +- .../repin_dockerfiles.sh | 80 ++-- bazel/resmoke/volatile_status.sh | 2 +- bazel/run_under_working_dir.sh | 4 +- buildscripts/BUILD.bazel | 12 - buildscripts/auto_install_db_contrib_tool.sh | 94 ++--- buildscripts/clangd_vscode.sh | 28 +- buildscripts/consolidate-repos-enterprise.sh | 155 ++++---- buildscripts/consolidate-repos.sh | 155 ++++---- buildscripts/mount_drives.sh | 240 ++++++------ buildscripts/poetry_sync.sh | 10 +- buildscripts/setup_engflow_creds.sh | 36 +- buildscripts/shellscripts-linters.sh | 41 --- ...import_script_with_mongo_prefix_version.sh | 1 - .../inputs/kafka_missing_version_import.sh | 14 +- .../sbom_linter/inputs/kafka_valid_import.sh | 14 +- .../inputs/kafka_wrong_version_import.sh | 14 +- buildscripts/yamllinters.sh | 8 +- etc/set_up_workstation.sh | 27 +- evergreen/BUILD.bazel | 5 - evergreen/antithesis_image_build_and_push.sh | 38 +- evergreen/bazel.sh | 6 +- evergreen/bazel_RBE_supported.sh | 14 +- evergreen/bazel_compile.sh | 144 ++++---- evergreen/bazel_coverage.sh | 6 +- evergreen/bazel_run.sh | 38 +- evergreen/bazel_test.sh | 66 ++-- evergreen/bazel_utility_functions.sh | 52 +-- evergreen/check_idl_compat.sh | 32 +- evergreen/check_legacy_command_types.sh | 2 +- evergreen/check_resmoke_failure.sh | 2 +- .../check_run_tests_infrastructure_failure.sh | 2 +- evergreen/check_workstation_setup.sh | 62 ++-- evergreen/cleanup_environment.sh | 16 +- evergreen/compile_ninja.sh | 6 +- evergreen/compiled_binaries_get.sh | 18 +- evergreen/consolidate_benchmark_txt.sh | 4 +- evergreen/container_registry_login.sh | 8 +- evergreen/coverity_build.sh | 10 +- evergreen/crypt_run_tests.sh | 26 +- .../do_jepsen_setup/create_fuzz_config.sh | 2 +- evergreen/do_jepsen_setup/nodes.sh | 4 +- evergreen/engflow_workspace_status.sh | 4 +- evergreen/external_auth_aws_setup.sh | 4 +- evergreen/external_auth_azure_setup.sh | 8 +- evergreen/external_auth_azure_teardown.sh | 16 +- evergreen/external_auth_gcp_setup.sh | 8 +- evergreen/external_auth_gcp_teardown.sh | 22 +- evergreen/external_auth_oidc_setup.sh | 4 +- evergreen/external_auth_oidc_teardown.sh | 8 +- evergreen/failed_unittests_gather.sh | 134 +++---- evergreen/feature_flag_tags_check.sh | 4 +- evergreen/full_disk_debug.sh | 20 +- evergreen/functions/add_git_tag.sh | 18 +- evergreen/functions/bolt.sh | 6 +- evergreen/functions/credentials_setup.sh | 4 +- evergreen/functions/download_pgo_profile.sh | 6 +- .../evergreen_api_credentials_configure.sh | 4 +- evergreen/functions/files_remove.sh | 12 +- evergreen/functions/fuse_watchdog_cleanup.sh | 4 +- evergreen/functions/get_mongot_version.sh | 2 +- .../functions/modified_patch_files_get_all.sh | 18 +- .../notary_client_credentials_setup.sh | 6 +- .../functions/process_threads_monitor.sh | 74 ++-- .../functions/remote_credentials_setup.sh | 22 +- evergreen/functions/resmoke_jobs_determine.sh | 14 +- evergreen/functions/run_diskstats.sh | 22 +- .../functions/system_resource_info_collect.sh | 2 +- evergreen/functions/task_timeout_determine.sh | 46 +-- evergreen/functions/tmp_directory_umount.sh | 2 +- evergreen/functions/ulimit_info_collect.sh | 12 +- evergreen/functions/venv_adjust.sh | 42 +-- evergreen/functions/venv_setup.sh | 50 +-- .../functions/version_expansions_generate.sh | 6 +- evergreen/functions/wiredtiger_develop_use.sh | 14 +- evergreen/garasign_gpg_crypt_sign.sh | 18 +- evergreen/garasign_gpg_sign.sh | 36 +- evergreen/garasign_jsign_sign.sh | 36 +- evergreen/gather_mongo_coredumps.sh | 12 +- evergreen/gen_feature_flags.sh | 2 +- evergreen/gen_supplementary_data.sh | 120 +++--- evergreen/gen_tasks_activate.sh | 6 +- .../generate_buildid_debug_symbols_mapping.sh | 16 +- ...erate_custom_build_promotion_expansions.sh | 10 +- evergreen/generate_evergreen_bazelrc.sh | 68 ++-- evergreen/generate_sast_report.sh | 26 +- evergreen/generate_version.sh | 18 +- evergreen/generate_version_burn_in.sh | 20 +- evergreen/get_all_resmoke_suite_configs.sh | 6 +- evergreen/get_bin_and_fcv_versions.sh | 2 +- evergreen/get_resmoke_suite_config.sh | 4 +- evergreen/hang_analyzer.sh | 8 +- evergreen/idl_tests_run.sh | 2 +- evergreen/jepsen_docker/cleanup.sh | 8 +- evergreen/jepsen_docker/docker-up.sh | 2 +- evergreen/jepsen_docker/list-append.sh | 76 ++-- .../jepsen_docker/setup_config_fuzzer.sh | 2 +- evergreen/jepsen_report.sh | 30 +- evergreen/jepsen_test_fail.sh | 4 +- evergreen/jepsen_test_run.sh | 34 +- evergreen/jstestfuzz_minimize.sh | 4 +- evergreen/jstestfuzz_run.sh | 66 ++-- evergreen/jstestfuzz_setup.sh | 20 +- evergreen/kill_processes.sh | 70 ++-- evergreen/lint_fuzzer_sanity_all.sh | 8 +- evergreen/lint_fuzzer_sanity_patch.sh | 2 +- evergreen/lint_shellscripts.sh | 14 - evergreen/lint_yaml.sh | 2 +- evergreen/local_client_logs_tar.sh | 6 +- evergreen/modify_debug_symbols.sh | 2 +- evergreen/monitor_build_status_run.sh | 4 +- evergreen/move_npm_logs.sh | 6 +- .../multiversion_exclude_tags_generate.sh | 18 +- evergreen/multiversion_selection.sh | 28 +- evergreen/multiversion_setup.sh | 2 +- evergreen/package.sh | 98 ++--- evergreen/packager.py_run.sh | 14 +- evergreen/packager_crypt_py_run.sh | 22 +- evergreen/packages_publish.sh | 14 +- evergreen/papertrail_generate_expansions.sh | 4 +- evergreen/perf-submission.sh | 18 +- evergreen/powercycle_check_host.sh | 34 +- evergreen/powercycle_exit.sh | 12 +- evergreen/powercycle_run_test.sh | 8 +- evergreen/powercycle_save_artifacts.sh | 4 +- evergreen/powercycle_sentinel_run.sh | 2 +- evergreen/powercycle_setup_host.sh | 2 +- evergreen/powercycle_ssh_failure_exit.sh | 6 +- evergreen/powercycle_tasks_generate.sh | 2 +- evergreen/prelude.sh | 94 ++--- evergreen/prelude_db_contrib_tool.sh | 104 +++--- evergreen/prelude_mongo_task_generator.sh | 8 +- evergreen/prelude_python.sh | 34 +- evergreen/prelude_system_env_variables.sh | 18 +- evergreen/prelude_venv.sh | 90 ++--- evergreen/prelude_workdir.sh | 28 +- evergreen/query_tester_test_repo_setup.sh | 24 +- .../query_tester_test_sparse_checkout.sh | 30 +- .../randomized_multiversion_tasks_generate.sh | 2 +- evergreen/record_mongodb_server_version.sh | 6 +- evergreen/resmoke_tests_execute.sh | 342 +++++++++--------- evergreen/resmoke_tests_execute_bazel.sh | 34 +- evergreen/resmoke_tests_runtime_validate.sh | 10 +- evergreen/restore_git_history_and_tags.sh | 34 +- evergreen/retry_git.sh | 22 +- evergreen/run_binary.sh | 2 +- evergreen/run_jstestfuzz/clone_repos.sh | 8 +- evergreen/run_python_script.sh | 2 +- evergreen/run_python_script_with_report.sh | 8 +- evergreen/run_upload_lock_push.sh | 64 ++-- evergreen/sasl_windows_cyrussasl_setup.sh | 6 +- evergreen/sasl_windows_cyrussasl_teardown.sh | 8 +- evergreen/selinux_run_test.sh | 108 +++--- evergreen/selinux_test_executor.sh | 116 +++--- evergreen/selinux_test_setup.sh | 32 +- evergreen/stitch_support_run_tests.sh | 2 +- evergreen/todos_check.sh | 12 +- evergreen/upload_custom_build_to_cloud_dev.sh | 72 ++-- evergreen/verify_build_output_present.sh | 2 +- evergreen/wait_for_resmoke_to_shutdown.sh | 36 +- evergreen/wiki_page.sh | 4 +- .../wiki_page_minimized_agg_query_fuzzer.sh | 8 +- ...ngo_binary_url_to_downstream_expansions.sh | 2 +- evergreen/write_sast_report_env_file.sh | 2 +- jstests/ssl/libs/localhost-cn-with-san.pem.sh | 10 +- jstests/ssl/x509/mkcrl.sh | 10 +- modules_poc/private_headers.sh | 6 +- .../aead_encryption_fle2_test_vectors.sh | 2 +- src/mongo/tla_plus/model-check.sh | 24 +- 174 files changed, 2259 insertions(+), 2387 deletions(-) create mode 100644 .editorconfig delete mode 100755 buildscripts/shellscripts-linters.sh delete mode 100644 evergreen/lint_shellscripts.sh diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000000..f8aad923a57 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*.sh] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5814ce118d7..35b4ce7af4a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,6 +10,7 @@ OWNERS.yml @10gen/server-root-ownership @svc-auto-approve-bot .bazel* @10gen/devprod-build @svc-auto-approve-bot .clang-format @10gen/server-programmability @svc-auto-approve-bot .clang-tidy.in @10gen/server-programmability @svc-auto-approve-bot +/.editorconfig @10gen/devprod-build @svc-auto-approve-bot .git* @10gen/devprod-build @svc-auto-approve-bot .mypy.ini @10gen/devprod-build @10gen/devprod-correctness @svc-auto-approve-bot .prettierignore @10gen/devprod-correctness @svc-auto-approve-bot diff --git a/OWNERS.yml b/OWNERS.yml index cab1997d31f..8e9c5877bd6 100644 --- a/OWNERS.yml +++ b/OWNERS.yml @@ -18,6 +18,9 @@ filters: - ".clang-tidy.in": approvers: - 10gen/server-programmability + - "/.editorconfig": + approvers: + - 10gen/devprod-build - ".git*": approvers: - 10gen/devprod-build diff --git a/bazel/format/BUILD.bazel b/bazel/format/BUILD.bazel index b82e7ccffe9..6d707e35517 100644 --- a/bazel/format/BUILD.bazel +++ b/bazel/format/BUILD.bazel @@ -6,8 +6,6 @@ py_binary( args = [ "--prettier", "$(location //:prettier)", - "--shellscripts-linters", - "$(location //buildscripts:shellscripts_linters)", "--rules-lint-format", "$(location :rules_lint_format)", "--rules-lint-format-check", @@ -17,7 +15,6 @@ py_binary( ":rules_lint_format", ":rules_lint_format.check", "//:prettier", - "//buildscripts:shellscripts_linters", "@shfmt", ], env = { @@ -38,9 +35,8 @@ format_multirun( graphql = "//:prettier", html = "//:prettier", markdown = "//:prettier", + shell = "@shfmt//:shfmt", sql = "//:prettier", starlark = "@buildifier_prebuilt//:buildifier", visibility = ["//visibility:public"], - # TODO(SERVER-101034): Enable rules_lint shfmt after sh files are reformatted with .editorconfig - # shell = "@shfmt//:shfmt", ) diff --git a/bazel/format/rules_lint_format_wrapper.py b/bazel/format/rules_lint_format_wrapper.py index 99c217b5e5c..d66f908b1e7 100644 --- a/bazel/format/rules_lint_format_wrapper.py +++ b/bazel/format/rules_lint_format_wrapper.py @@ -82,21 +82,6 @@ def run_rules_lint( return True -def run_shellscripts_linters(shellscripts_linters: pathlib.Path, check: bool) -> bool: - try: - command = [str(shellscripts_linters)] - if not check: - print("Running shellscripts formatter") - command.append("fix") - else: - print("Running shellscripts linter") - repo_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - subprocess.run(command, check=True, env=os.environ, cwd=repo_path) - except subprocess.CalledProcessError: - return False - return True - - def run_prettier( prettier: pathlib.Path, check: bool, files_to_format: Union[List[str], str] = "all" ) -> bool: @@ -169,12 +154,6 @@ def main() -> int: parser.add_argument( "--prettier", help="Set the path to prettier", required=True, type=pathlib.Path ) - parser.add_argument( - "--shellscripts-linters", - help="Set the path to shellscripts_linters", - required=True, - type=pathlib.Path, - ) parser.add_argument( "--rules-lint-format", help="Set the path to rules_lint's formatter", @@ -200,7 +179,6 @@ def main() -> int: args = parser.parse_args() prettier_path: pathlib.Path = args.prettier.resolve() - shellscripts_linters_path: pathlib.Path = args.shellscripts_linters.resolve() os.chdir(default_dir) @@ -235,7 +213,6 @@ def main() -> int: if run_rules_lint( args.rules_lint_format, args.rules_lint_format_check, args.check, files_to_format ) - and run_shellscripts_linters(shellscripts_linters_path, args.check) and run_prettier(prettier_path, args.check, files_to_format) else 1 ) diff --git a/bazel/lint.sh b/bazel/lint.sh index 186717a4cdb..c3554b71a69 100755 --- a/bazel/lint.sh +++ b/bazel/lint.sh @@ -5,8 +5,8 @@ GREEN='\033[0;32m' NO_COLOR='\033[0m' if [[ $1 == "ALL_PASSING" ]]; then - echo -e "${GREEN}INFO:${NO_COLOR} No linter errors found!" - exit 0 + echo -e "${GREEN}INFO:${NO_COLOR} No linter errors found!" + exit 0 fi echo -e "${RED}ERROR:${NO_COLOR} Linter run failed, see details above" diff --git a/bazel/remote_execution_container/repin_dockerfiles.sh b/bazel/remote_execution_container/repin_dockerfiles.sh index adbaedc38de..617890e4a03 100755 --- a/bazel/remote_execution_container/repin_dockerfiles.sh +++ b/bazel/remote_execution_container/repin_dockerfiles.sh @@ -153,34 +153,34 @@ get_package_versions() { pkg_manager=$(get_package_manager "$image") case "$pkg_manager" in - yum) - docker run --rm "$image" bash -c " + yum) + docker run --rm "$image" bash -c " yum info ${packages[*]} 2>/dev/null | awk '/^Name/ {name=\$3} /^Version/ {version=\$3} /^Release/ {release=\$3} /^Release/ {print name \"-\" version \"-\" release}' | sort -u" - ;; - apt) - docker run --rm "$image" bash -c " + ;; + apt) + docker run --rm "$image" bash -c " apt-get update >/dev/null 2>&1 && apt-cache policy ${packages[*]} | awk '/^[^ ]/ {pkg=\$1} /Candidate:/ {print pkg \"=\" \$2}' | sort -u" - ;; - zypper) - # TODO(SERVER-93423): Pin suse package versions. At the moment this - # breaks the remote_execution_containers_generator.py script. - printf '%s\n' "${packages[@]}" | sort -u - # docker run --rm "$image" bash -c " - # zypper --non-interactive refresh >/dev/null 2>&1 && - # zypper --non-interactive info ${packages[*]} | - # awk '/^Name/ {name=\$3} /^Version/ {version=\$3} /^Version/ {print name \"=\" version}' | - # sort -u" - ;; - *) - echo "Unsupported package manager for image: $image" >&2 - return 1 - ;; + ;; + zypper) + # TODO(SERVER-93423): Pin suse package versions. At the moment this + # breaks the remote_execution_containers_generator.py script. + printf '%s\n' "${packages[@]}" | sort -u + # docker run --rm "$image" bash -c " + # zypper --non-interactive refresh >/dev/null 2>&1 && + # zypper --non-interactive info ${packages[*]} | + # awk '/^Name/ {name=\$3} /^Version/ {version=\$3} /^Version/ {print name \"=\" version}' | + # sort -u" + ;; + *) + echo "Unsupported package manager for image: $image" >&2 + return 1 + ;; esac } @@ -207,25 +207,25 @@ generate_dockerfile() { install_lines=$(get_package_versions "$image" "${packages[@]}" | sed 's/^/ /' | sed 's/$/\ \\/') case "$pkg_manager" in - yum) - update_cmd="yum check-update || true" - install_cmd="yum install -y" - clean_cmd="&& yum clean all && rm -rf /var/cache/yum/*" - ;; - apt) - update_cmd="apt-get update" - install_cmd="DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends" - clean_cmd="&& rm -rf /var/lib/apt/lists/*" - ;; - zypper) - update_cmd="zypper refresh" - install_cmd="zypper install -y --no-recommends" - clean_cmd="&& zypper clean --all" - ;; - *) - echo "Unsupported package manager for image: $image" >&2 - return 1 - ;; + yum) + update_cmd="yum check-update || true" + install_cmd="yum install -y" + clean_cmd="&& yum clean all && rm -rf /var/cache/yum/*" + ;; + apt) + update_cmd="apt-get update" + install_cmd="DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends" + clean_cmd="&& rm -rf /var/lib/apt/lists/*" + ;; + zypper) + update_cmd="zypper refresh" + install_cmd="zypper install -y --no-recommends" + clean_cmd="&& zypper clean --all" + ;; + *) + echo "Unsupported package manager for image: $image" >&2 + return 1 + ;; esac # Remove colons from package versions for Debian and Ubuntu @@ -234,7 +234,7 @@ generate_dockerfile() { fi mkdir -p "$output_dir" - cat << EOF > "$output_dir/dockerfile" + cat <"$output_dir/dockerfile" # DO NOT EDIT. # # This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin diff --git a/bazel/resmoke/volatile_status.sh b/bazel/resmoke/volatile_status.sh index 42b47ee3819..ce148154d9c 100755 --- a/bazel/resmoke/volatile_status.sh +++ b/bazel/resmoke/volatile_status.sh @@ -3,7 +3,7 @@ # This script is used as a workspace status command # bazel test --workspace_status_command=bazel/resmoke/volatile_status.sh # to populate key-value pairs in bazel-out/volatile-status.txt. -# This file and the key-values can be consumed by bazel rules, but bazel +# This file and the key-values can be consumed by bazel rules, but bazel # pretends this file never changes when deciding what to rebuild. # Evergreen expansions used primarily for Resmoke telemetry diff --git a/bazel/run_under_working_dir.sh b/bazel/run_under_working_dir.sh index 370b267bcf7..8123b10b9c4 100755 --- a/bazel/run_under_working_dir.sh +++ b/bazel/run_under_working_dir.sh @@ -5,8 +5,8 @@ set -e RUNFILES_WORKING_DIRECTORY="$(pwd)" if [ -z $BUILD_WORKING_DIRECTORY ]; then - echo "ERROR: BUILD_WORKING_DIRECTORY was not set, was this run from bazel?" - exit 1 + echo "ERROR: BUILD_WORKING_DIRECTORY was not set, was this run from bazel?" + exit 1 fi cd $BUILD_WORKING_DIRECTORY diff --git a/buildscripts/BUILD.bazel b/buildscripts/BUILD.bazel index d65794f3b7b..7b62fbf0e14 100644 --- a/buildscripts/BUILD.bazel +++ b/buildscripts/BUILD.bazel @@ -111,18 +111,6 @@ py_binary( ], ) -sh_binary( - name = "shellscripts_linters", - srcs = ["shellscripts-linters.sh"], - data = [ - "@shfmt", - ], - env = { - "SHFMT_PATH": "$(rootpath @shfmt//:shfmt)", - }, - visibility = ["//visibility:public"], -) - py_library( name = "mongo_toolchain", srcs = [ diff --git a/buildscripts/auto_install_db_contrib_tool.sh b/buildscripts/auto_install_db_contrib_tool.sh index eafece65d0c..5572727c379 100755 --- a/buildscripts/auto_install_db_contrib_tool.sh +++ b/buildscripts/auto_install_db_contrib_tool.sh @@ -8,70 +8,70 @@ echo "+------------------------------------------------------------------------- echo if [[ -d "/opt/mongodbtoolchain/v4/bin" ]]; then - export PATH="/opt/mongodbtoolchain/v4/bin:$PATH" + export PATH="/opt/mongodbtoolchain/v4/bin:$PATH" fi if [[ -d "/opt/mongodbtoolchain/v5/bin" ]]; then - export PATH="/opt/mongodbtoolchain/v5/bin:$PATH" + export PATH="/opt/mongodbtoolchain/v5/bin:$PATH" fi rc_file="" if [[ -f "$HOME/.bashrc" ]]; then - rc_file="$HOME/.bashrc" + rc_file="$HOME/.bashrc" fi if [[ -f "$HOME/.zshrc" ]]; then - rc_file="$HOME/.zshrc" + rc_file="$HOME/.zshrc" fi -if ! command -v db-contrib-tool &> /dev/null; then - if ! python3 -c "import sys; sys.exit(sys.version_info < (3, 7))" &> /dev/null; then - actual_version=$(python3 -c 'import sys; print(sys.version)') - echo "You must have python3.7+ installed. Detected version $actual_version." - echo "To avoid unexpected issues, python3.7+ will not be automatically installed." - echo "Please, do it yourself." - echo - echo "On macOS you can run:" - echo - echo " brew install python3" - echo - exit 1 - fi - - if command -v pipx &> /dev/null; then - echo "Found pipx: $(command -v pipx)." - echo "Using it to install 'db-contrib-tool'." - echo - - pipx ensurepath &> /dev/null - if [[ -f "$rc_file" ]]; then - source "$rc_file" +if ! command -v db-contrib-tool &>/dev/null; then + if ! python3 -c "import sys; sys.exit(sys.version_info < (3, 7))" &>/dev/null; then + actual_version=$(python3 -c 'import sys; print(sys.version)') + echo "You must have python3.7+ installed. Detected version $actual_version." + echo "To avoid unexpected issues, python3.7+ will not be automatically installed." + echo "Please, do it yourself." + echo + echo "On macOS you can run:" + echo + echo " brew install python3" + echo + exit 1 fi - pipx install db-contrib-tool --python $(command -v python3) --force - echo - else - if ! python3 -m pipx --version &> /dev/null; then - echo "Couldn't find pipx. Installing it as python3 module:" - echo " $(command -v python3) -m pip install pipx" - echo - python3 -m pip install pipx - echo + if command -v pipx &>/dev/null; then + echo "Found pipx: $(command -v pipx)." + echo "Using it to install 'db-contrib-tool'." + echo + + pipx ensurepath &>/dev/null + if [[ -f "$rc_file" ]]; then + source "$rc_file" + fi + + pipx install db-contrib-tool --python $(command -v python3) --force + echo else - echo "Found pipx installed as python3 module:" - echo " $(command -v python3) -m pipx --version" - echo "Using it to install 'db-contrib-tool'." - echo - fi + if ! python3 -m pipx --version &>/dev/null; then + echo "Couldn't find pipx. Installing it as python3 module:" + echo " $(command -v python3) -m pip install pipx" + echo + python3 -m pip install pipx + echo + else + echo "Found pipx installed as python3 module:" + echo " $(command -v python3) -m pipx --version" + echo "Using it to install 'db-contrib-tool'." + echo + fi - python3 -m pipx ensurepath &> /dev/null - if [[ -f "$rc_file" ]]; then - source "$rc_file" - fi + python3 -m pipx ensurepath &>/dev/null + if [[ -f "$rc_file" ]]; then + source "$rc_file" + fi - python3 -m pipx install db-contrib-tool --force - echo - fi + python3 -m pipx install db-contrib-tool --force + echo + fi fi echo "Please, open a new shell or run:" diff --git a/buildscripts/clangd_vscode.sh b/buildscripts/clangd_vscode.sh index 4dd110d26be..a7b58ec40e8 100755 --- a/buildscripts/clangd_vscode.sh +++ b/buildscripts/clangd_vscode.sh @@ -5,31 +5,31 @@ ARGS=("$@") # Ordered list of possible clangd locations CANDIDATES=( - "$(command -v custom-clangd)" - "$(find .compiledb/compiledb-*/external/mongo_toolchain_v5/v5/bin/clangd)" - "$(find bazel-*/external/mongo_toolchain_v5/v5/bin/clangd)" - "/opt/mongodbtoolchain/v5/bin/clangd" + "$(command -v custom-clangd)" + "$(find .compiledb/compiledb-*/external/mongo_toolchain_v5/v5/bin/clangd)" + "$(find bazel-*/external/mongo_toolchain_v5/v5/bin/clangd)" + "/opt/mongodbtoolchain/v5/bin/clangd" ) # Find the first available clangd for CANDIDATE in "${CANDIDATES[@]}"; do - if [[ -x "$CANDIDATE" ]]; then - CLANGD="$CANDIDATE" - echo "[INFO] Using clangd at: $CLANGD" >&2 - break - fi + if [[ -x "$CANDIDATE" ]]; then + CLANGD="$CANDIDATE" + echo "[INFO] Using clangd at: $CLANGD" >&2 + break + fi done # Fail if no clangd was found if [[ -z "$CLANGD" ]]; then - echo "[ERROR] clangd not found in any of the expected locations." >&2 - exit 1 + echo "[ERROR] clangd not found in any of the expected locations." >&2 + exit 1 fi FINAL_ARGS=( - "${ARGS[@]}" - "--query-driver=./**/*{clang,gcc,g++}*" # allow any clang or gcc binary in the repo - "--header-insertion=never" + "${ARGS[@]}" + "--query-driver=./**/*{clang,gcc,g++}*" # allow any clang or gcc binary in the repo + "--header-insertion=never" ) # Log the full command (optional) diff --git a/buildscripts/consolidate-repos-enterprise.sh b/buildscripts/consolidate-repos-enterprise.sh index 93e02eee4a0..ea70b9e0294 100755 --- a/buildscripts/consolidate-repos-enterprise.sh +++ b/buildscripts/consolidate-repos-enterprise.sh @@ -26,87 +26,75 @@ mkdir -p "$repodir/yum/redhat" # to support different $releasever values in yum repo configurations # -if [ ! -e "$repodir/yum/redhat/7Server" ] -then - ln -s 7 "$repodir/yum/redhat/7Server" +if [ ! -e "$repodir/yum/redhat/7Server" ]; then + ln -s 7 "$repodir/yum/redhat/7Server" fi -if [ ! -e "$repodir/yum/redhat/6Server" ] -then - ln -s 6 "$repodir/yum/redhat/6Server" +if [ ! -e "$repodir/yum/redhat/6Server" ]; then + ln -s 6 "$repodir/yum/redhat/6Server" fi -if [ ! -e "$repodir/yum/redhat/5Server" ] -then - ln -s 5 "$repodir/yum/redhat/5Server" +if [ ! -e "$repodir/yum/redhat/5Server" ]; then + ln -s 5 "$repodir/yum/redhat/5Server" fi echo "Scanning and copying package files from $source_dir" echo ". = skipping existing file, @ = copying file" -for package in $(find "$source_dir" -not \( -path "$repodir" -prune \) -and \( -name \*.rpm -o -name \*.deb -o -name Release \)) -do - new_package_location="$repodir$(echo "$package" | sed 's/\/var\/www-enterprise\/[^\/]*//;')" - # skip if the directory structure looks weird - # - if echo "$new_package_location" | grep -q /repo/ - then - continue - fi +for package in $(find "$source_dir" -not \( -path "$repodir" -prune \) -and \( -name \*.rpm -o -name \*.deb -o -name Release \)); do + new_package_location="$repodir$(echo "$package" | sed 's/\/var\/www-enterprise\/[^\/]*//;')" + # skip if the directory structure looks weird + # + if echo "$new_package_location" | grep -q /repo/; then + continue + fi - # skip if not enterprise package - # - if ! echo "$new_package_location" | grep -q enterprise - then - continue - fi - # skip if it's already there - # - if [ -e "$new_package_location" -a "$(basename "$package")" != "Release" ] - then - echo -n . - else - mkdir -p "$(dirname "$new_package_location")" - echo -n @ - cp "$package" "$new_package_location" - fi + # skip if not enterprise package + # + if ! echo "$new_package_location" | grep -q enterprise; then + continue + fi + # skip if it's already there + # + if [ -e "$new_package_location" -a "$(basename "$package")" != "Release" ]; then + echo -n . + else + mkdir -p "$(dirname "$new_package_location")" + echo -n @ + cp "$package" "$new_package_location" + fi done echo # packages are in place, now create metadata # -for debian_dir in "$repodir"/apt/ubuntu "$repodir"/apt/debian -do - cd "$debian_dir" - for section_dir in $(find dists -type d -name multiverse -o -name main) - do - for arch_dir in "$section_dir"/{binary-i386,binary-amd64} - do - echo "Generating Packages file under $debian_dir/$arch_dir" - if [ ! -d $arch_dir ] - then - mkdir $arch_dir - fi - dpkg-scanpackages --multiversion "$arch_dir" > "$arch_dir"/Packages - gzip -9c "$arch_dir"/Packages > "$arch_dir"/Packages.gz +for debian_dir in "$repodir"/apt/ubuntu "$repodir"/apt/debian; do + cd "$debian_dir" + for section_dir in $(find dists -type d -name multiverse -o -name main); do + for arch_dir in "$section_dir"/{binary-i386,binary-amd64}; do + echo "Generating Packages file under $debian_dir/$arch_dir" + if [ ! -d $arch_dir ]; then + mkdir $arch_dir + fi + dpkg-scanpackages --multiversion "$arch_dir" >"$arch_dir"/Packages + gzip -9c "$arch_dir"/Packages >"$arch_dir"/Packages.gz + done done - done - for release_file in $(find "$debian_dir" -name Release) - do - release_dir=$(dirname "$release_file") - echo "Generating Release file under $release_dir" - cd $release_dir - tempfile=$(mktemp /tmp/ReleaseXXXXXX) - tempfile2=$(mktemp /tmp/ReleaseXXXXXX) - mv Release $tempfile - head -7 $tempfile > $tempfile2 - apt-ftparchive release . >> $tempfile2 - cp $tempfile2 Release - chmod 644 Release - rm Release.gpg - echo "Signing Release file" - gpg -r "$gpg_recip" --no-secmem-warning -abs --output Release.gpg Release - done + for release_file in $(find "$debian_dir" -name Release); do + release_dir=$(dirname "$release_file") + echo "Generating Release file under $release_dir" + cd $release_dir + tempfile=$(mktemp /tmp/ReleaseXXXXXX) + tempfile2=$(mktemp /tmp/ReleaseXXXXXX) + mv Release $tempfile + head -7 $tempfile >$tempfile2 + apt-ftparchive release . >>$tempfile2 + cp $tempfile2 Release + chmod 644 Release + rm Release.gpg + echo "Signing Release file" + gpg -r "$gpg_recip" --no-secmem-warning -abs --output Release.gpg Release + done done # Create symlinks for stable and unstable branches @@ -118,29 +106,24 @@ done # /var/www-enterprise/repo.consolidated/apt/ubuntu/dists/precise/mongodb-enterprise/unstable -> 2.5 # /var/www-enterprise/repo.consolidated/apt/debian/dists/wheezy/mongodb-enterprise/unstable -> 2.5 # -for unstable_branch_dir in "$repodir"/yum/redhat/*/*/$unstable_branch "$repodir"/yum/amazon/*/*/$unstable_branch "$repodir"/apt/debian/dists/*/*/$unstable_branch "$repodir"/apt/ubuntu/dists/*/*/$unstable_branch "$repodir"/zypper/suse/*/*/$unstable_branch -do - full_unstable_path=$(dirname "$unstable_branch_dir")/unstable - if [ -e "$unstable_branch_dir" -a ! -e "$full_unstable_path" ] - then - echo "Linking unstable branch directory $unstable_branch_dir to $full_unstable_path" - ln -s $unstable_branch $full_unstable_path - fi +for unstable_branch_dir in "$repodir"/yum/redhat/*/*/$unstable_branch "$repodir"/yum/amazon/*/*/$unstable_branch "$repodir"/apt/debian/dists/*/*/$unstable_branch "$repodir"/apt/ubuntu/dists/*/*/$unstable_branch "$repodir"/zypper/suse/*/*/$unstable_branch; do + full_unstable_path=$(dirname "$unstable_branch_dir")/unstable + if [ -e "$unstable_branch_dir" -a ! -e "$full_unstable_path" ]; then + echo "Linking unstable branch directory $unstable_branch_dir to $full_unstable_path" + ln -s $unstable_branch $full_unstable_path + fi done -for stable_branch_dir in "$repodir"/yum/redhat/*/*/$stable_branch "$repodir"/yum/amazon/*/*/$stable_branch "$repodir"/apt/debian/dists/*/*/$stable_branch "$repodir"/apt/ubuntu/dists/*/*/$stable_branch "$repodir"/zypper/suse/*/*/$stable_branch -do - full_stable_path=$(dirname "$stable_branch_dir")/stable - if [ -e "$stable_branch_dir" -a ! -e "$full_stable_path" ] - then - echo "Linking stable branch directory $stable_branch_dir to $full_stable_path" - ln -s $stable_branch $full_stable_path - fi +for stable_branch_dir in "$repodir"/yum/redhat/*/*/$stable_branch "$repodir"/yum/amazon/*/*/$stable_branch "$repodir"/apt/debian/dists/*/*/$stable_branch "$repodir"/apt/ubuntu/dists/*/*/$stable_branch "$repodir"/zypper/suse/*/*/$stable_branch; do + full_stable_path=$(dirname "$stable_branch_dir")/stable + if [ -e "$stable_branch_dir" -a ! -e "$full_stable_path" ]; then + echo "Linking stable branch directory $stable_branch_dir to $full_stable_path" + ln -s $stable_branch $full_stable_path + fi done -for rpm_dir in $(find "$repodir"/yum/redhat "$repodir"/zypper/suse -type d -name x86_64 -o -name i386) -do - echo "Generating redhat repo metadata under $rpm_dir" - cd "$rpm_dir" - createrepo . +for rpm_dir in $(find "$repodir"/yum/redhat "$repodir"/zypper/suse -type d -name x86_64 -o -name i386); do + echo "Generating redhat repo metadata under $rpm_dir" + cd "$rpm_dir" + createrepo . done diff --git a/buildscripts/consolidate-repos.sh b/buildscripts/consolidate-repos.sh index c0c62b5a87b..9290d9c184d 100755 --- a/buildscripts/consolidate-repos.sh +++ b/buildscripts/consolidate-repos.sh @@ -26,87 +26,75 @@ mkdir -p "$repodir/yum/redhat" # to support different $releasever values in yum repo configurations # -if [ ! -e "$repodir/yum/redhat/6Server" ] -then - ln -s 6 "$repodir/yum/redhat/6Server" +if [ ! -e "$repodir/yum/redhat/6Server" ]; then + ln -s 6 "$repodir/yum/redhat/6Server" fi -if [ ! -e "$repodir/yum/redhat/7Server" ] -then - ln -s 7 "$repodir/yum/redhat/7Server" +if [ ! -e "$repodir/yum/redhat/7Server" ]; then + ln -s 7 "$repodir/yum/redhat/7Server" fi -if [ ! -e "$repodir/yum/redhat/5Server" ] -then - ln -s 5 "$repodir/yum/redhat/5Server" +if [ ! -e "$repodir/yum/redhat/5Server" ]; then + ln -s 5 "$repodir/yum/redhat/5Server" fi echo "Scanning and copying package files from $source_dir" echo ". = skipping existing file, @ = copying file" -for package in $(find "$source_dir" -not \( -path "$repodir" -prune \) -and \( -name \*.rpm -o -name \*.deb -o -name Release \)) -do - new_package_location="$repodir$(echo "$package" | sed 's/\/var\/www-org\/[^\/]*//;')" - # skip if the directory structure looks weird - # - if echo "$new_package_location" | grep -q /repo/ - then - continue - fi +for package in $(find "$source_dir" -not \( -path "$repodir" -prune \) -and \( -name \*.rpm -o -name \*.deb -o -name Release \)); do + new_package_location="$repodir$(echo "$package" | sed 's/\/var\/www-org\/[^\/]*//;')" + # skip if the directory structure looks weird + # + if echo "$new_package_location" | grep -q /repo/; then + continue + fi - # skip if not community package - # - if ! echo "$new_package_location" | grep -q org - then - continue - fi - # skip if it's already there - # - if [ -e "$new_package_location" -a "$(basename "$package")" != "Release" ] - then - echo -n . - else - mkdir -p "$(dirname "$new_package_location")" - echo -n @ - cp "$package" "$new_package_location" - fi + # skip if not community package + # + if ! echo "$new_package_location" | grep -q org; then + continue + fi + # skip if it's already there + # + if [ -e "$new_package_location" -a "$(basename "$package")" != "Release" ]; then + echo -n . + else + mkdir -p "$(dirname "$new_package_location")" + echo -n @ + cp "$package" "$new_package_location" + fi done echo # packages are in place, now create metadata # -for debian_dir in "$repodir"/apt/ubuntu "$repodir"/apt/debian -do - cd "$debian_dir" - for section_dir in $(find dists -type d -name multiverse -o -name main) - do - for arch_dir in "$section_dir"/{binary-i386,binary-amd64} - do - echo "Generating Packages file under $debian_dir/$arch_dir" - if [ ! -d $arch_dir ] - then - mkdir $arch_dir - fi - dpkg-scanpackages --multiversion "$arch_dir" > "$arch_dir"/Packages - gzip -9c "$arch_dir"/Packages > "$arch_dir"/Packages.gz +for debian_dir in "$repodir"/apt/ubuntu "$repodir"/apt/debian; do + cd "$debian_dir" + for section_dir in $(find dists -type d -name multiverse -o -name main); do + for arch_dir in "$section_dir"/{binary-i386,binary-amd64}; do + echo "Generating Packages file under $debian_dir/$arch_dir" + if [ ! -d $arch_dir ]; then + mkdir $arch_dir + fi + dpkg-scanpackages --multiversion "$arch_dir" >"$arch_dir"/Packages + gzip -9c "$arch_dir"/Packages >"$arch_dir"/Packages.gz + done done - done - for release_file in $(find "$debian_dir" -name Release) - do - release_dir=$(dirname "$release_file") - echo "Generating Release file under $release_dir" - cd $release_dir - tempfile=$(mktemp /tmp/ReleaseXXXXXX) - tempfile2=$(mktemp /tmp/ReleaseXXXXXX) - mv Release $tempfile - head -7 $tempfile > $tempfile2 - apt-ftparchive release . >> $tempfile2 - cp $tempfile2 Release - chmod 644 Release - rm Release.gpg - echo "Signing Release file" - gpg -r "$gpg_recip" --no-secmem-warning -abs --output Release.gpg Release - done + for release_file in $(find "$debian_dir" -name Release); do + release_dir=$(dirname "$release_file") + echo "Generating Release file under $release_dir" + cd $release_dir + tempfile=$(mktemp /tmp/ReleaseXXXXXX) + tempfile2=$(mktemp /tmp/ReleaseXXXXXX) + mv Release $tempfile + head -7 $tempfile >$tempfile2 + apt-ftparchive release . >>$tempfile2 + cp $tempfile2 Release + chmod 644 Release + rm Release.gpg + echo "Signing Release file" + gpg -r "$gpg_recip" --no-secmem-warning -abs --output Release.gpg Release + done done # Create symlinks for stable and unstable branches @@ -118,29 +106,24 @@ done # /var/www-org/repo.consolidated/apt/ubuntu/dists/precise/mongodb-org/unstable -> 2.5 # /var/www-org/repo.consolidated/apt/debian/dists/wheezy/mongodb-org/unstable -> 2.5 # -for unstable_branch_dir in "$repodir"/yum/redhat/*/*/$unstable_branch "$repodir"/yum/amazon/*/*/$unstable_branch "$repodir"/apt/debian/dists/*/*/$unstable_branch "$repodir"/apt/ubuntu/dists/*/*/$unstable_branch "$repodir"/zypper/suse/*/*/$unstable_branch -do - full_unstable_path=$(dirname "$unstable_branch_dir")/unstable - if [ -e "$unstable_branch_dir" -a ! -e "$full_unstable_path" ] - then - echo "Linking unstable branch directory $unstable_branch_dir to $full_unstable_path" - ln -s $unstable_branch $full_unstable_path - fi +for unstable_branch_dir in "$repodir"/yum/redhat/*/*/$unstable_branch "$repodir"/yum/amazon/*/*/$unstable_branch "$repodir"/apt/debian/dists/*/*/$unstable_branch "$repodir"/apt/ubuntu/dists/*/*/$unstable_branch "$repodir"/zypper/suse/*/*/$unstable_branch; do + full_unstable_path=$(dirname "$unstable_branch_dir")/unstable + if [ -e "$unstable_branch_dir" -a ! -e "$full_unstable_path" ]; then + echo "Linking unstable branch directory $unstable_branch_dir to $full_unstable_path" + ln -s $unstable_branch $full_unstable_path + fi done -for stable_branch_dir in "$repodir"/yum/redhat/*/*/$stable_branch "$repodir"/yum/amazon/*/*/$stable_branch "$repodir"/apt/debian/dists/*/*/$stable_branch "$repodir"/apt/ubuntu/dists/*/*/$stable_branch "$repodir"/zypper/suse/*/*/$stable_branch -do - full_stable_path=$(dirname "$stable_branch_dir")/stable - if [ -e "$stable_branch_dir" -a ! -e "$full_stable_path" ] - then - echo "Linking stable branch directory $stable_branch_dir to $full_stable_path" - ln -s $stable_branch $full_stable_path - fi +for stable_branch_dir in "$repodir"/yum/redhat/*/*/$stable_branch "$repodir"/yum/amazon/*/*/$stable_branch "$repodir"/apt/debian/dists/*/*/$stable_branch "$repodir"/apt/ubuntu/dists/*/*/$stable_branch "$repodir"/zypper/suse/*/*/$stable_branch; do + full_stable_path=$(dirname "$stable_branch_dir")/stable + if [ -e "$stable_branch_dir" -a ! -e "$full_stable_path" ]; then + echo "Linking stable branch directory $stable_branch_dir to $full_stable_path" + ln -s $stable_branch $full_stable_path + fi done -for rpm_dir in $(find "$repodir"/yum/redhat "$repodir"/yum/amazon "$repodir"/zypper/suse -type d -name x86_64 -o -name i386) -do - echo "Generating redhat repo metadata under $rpm_dir" - cd "$rpm_dir" - createrepo . +for rpm_dir in $(find "$repodir"/yum/redhat "$repodir"/yum/amazon "$repodir"/zypper/suse -type d -name x86_64 -o -name i386); do + echo "Generating redhat repo metadata under $rpm_dir" + cd "$rpm_dir" + createrepo . done diff --git a/buildscripts/mount_drives.sh b/buildscripts/mount_drives.sh index 0070d9b0abc..e7f963bba2d 100755 --- a/buildscripts/mount_drives.sh +++ b/buildscripts/mount_drives.sh @@ -12,7 +12,7 @@ user_group=$USER:$(id -Gn $USER | cut -f1 -d ' ') # _usage_: Provides usage infomation function _usage_ { - cat << EOF + cat <, REQUIRED, Space separated list of devices to mount /data on, @@ -31,150 +31,144 @@ This script supports the following parameters for Windows & Linux platforms: EOF } - # Parse command line options -while getopts "d:l:o:r:t:u:?" option -do - case $option in +while getopts "d:l:o:r:t:u:?" option; do + case $option in d) - data_device_names=$OPTARG - ;; + data_device_names=$OPTARG + ;; l) - log_device_name=$OPTARG - ;; + log_device_name=$OPTARG + ;; o) - mount_options=$OPTARG - ;; + mount_options=$OPTARG + ;; r) - data_raid_device_name=$OPTARG + data_raid_device_name=$OPTARG ;; t) - fs_type=$OPTARG - ;; + fs_type=$OPTARG + ;; u) - user_group=$OPTARG - ;; - \?|*) - _usage_ - exit 0 - ;; - esac + user_group=$OPTARG + ;; + \? | *) + _usage_ + exit 0 + ;; + esac done function mount_drive { - local root_dir=$1 - local sub_dirs=$2 - local device_names=$3 - local raid_device_name=$4 - local mount_options=$5 - local fs_type=$6 - local user_group=$7 + local root_dir=$1 + local sub_dirs=$2 + local device_names=$3 + local raid_device_name=$4 + local mount_options=$5 + local fs_type=$6 + local user_group=$7 - # Determine how many devices were specified - local num_devices=0 - for device_name in $device_names - do - local devices="$devices /dev/$device_name" - let num_devices=num_devices+1 - done + # Determine how many devices were specified + local num_devices=0 + for device_name in $device_names; do + local devices="$devices /dev/$device_name" + let num_devices=num_devices+1 + done - # $OS is defined in Cygwin - if [ "Windows_NT" = "$OS" ]; then - if [ $num_devices -ne 1 ]; then - echo "Must specify only one drive" - _usage_ - exit 1 - fi + # $OS is defined in Cygwin + if [ "Windows_NT" = "$OS" ]; then + if [ $num_devices -ne 1 ]; then + echo "Must specify only one drive" + _usage_ + exit 1 + fi - local drive_poll_retry=0 - local drive_poll_delay=0 - local drive_retry_max=40 + local drive_poll_retry=0 + local drive_poll_delay=0 + local drive_retry_max=40 - local drive=$device_names - local system_drive=c + local drive=$device_names + local system_drive=c - while true; - do - sleep $drive_poll_delay - echo "Looking for drive '$drive' to mount $root_dir" - if [ -d /cygdrive/$drive ]; then - echo "Found drive" - rm -rf /$root_dir - rm -rf /cygdrive/$system_drive/$root_dir - mkdir $drive:\\$root_dir - cmd.exe /c mklink /J $system_drive:\\$root_dir $drive:\\$root_dir - ln -s /cygdrive/$drive/$root_dir /$root_dir - setfacl -s user::rwx,group::rwx,other::rwx /cygdrive/$drive/$root_dir - for sub_dir in $sub_dirs - do - mkdir -p /cygdrive/$drive/$root_dir/$sub_dir + while true; do + sleep $drive_poll_delay + echo "Looking for drive '$drive' to mount $root_dir" + if [ -d /cygdrive/$drive ]; then + echo "Found drive" + rm -rf /$root_dir + rm -rf /cygdrive/$system_drive/$root_dir + mkdir $drive:\\$root_dir + cmd.exe /c mklink /J $system_drive:\\$root_dir $drive:\\$root_dir + ln -s /cygdrive/$drive/$root_dir /$root_dir + setfacl -s user::rwx,group::rwx,other::rwx /cygdrive/$drive/$root_dir + for sub_dir in $sub_dirs; do + mkdir -p /cygdrive/$drive/$root_dir/$sub_dir + done + chown -R $user_group /cygdrive/$system_drive/$root_dir + break + fi + let drive_poll_retry=drive_poll_retry+1 + if [ $drive_poll_retry -eq $drive_retry_max ]; then + echo "Timed out trying to mount $root_dir drive." + exit 1 + fi + let drive_poll_delay=drive_poll_delay+5 done - chown -R $user_group /cygdrive/$system_drive/$root_dir - break - fi - let drive_poll_retry=drive_poll_retry+1 - if [ $drive_poll_retry -eq $drive_retry_max ]; then - echo "Timed out trying to mount $root_dir drive." - exit 1 - fi - let drive_poll_delay=drive_poll_delay+5 - done - elif [ $(uname | awk '{print tolower($0)}') = "linux" ]; then - if [ $num_devices -eq 0 ]; then - echo "Must specify atleast one device" - _usage_ - exit 1 - elif [ $num_devices -gt 1 ]; then - if [ -z "$raid_device_name" ]; then - echo "Missing RAID device name" - _usage_ - exit 1 - fi - fi + elif [ $(uname | awk '{print tolower($0)}') = "linux" ]; then + if [ $num_devices -eq 0 ]; then + echo "Must specify atleast one device" + _usage_ + exit 1 + elif [ $num_devices -gt 1 ]; then + if [ -z "$raid_device_name" ]; then + echo "Missing RAID device name" + _usage_ + exit 1 + fi + fi - # Unmount the current devices, if already mounted - umount /mnt || true - umount $devices || true + # Unmount the current devices, if already mounted + umount /mnt || true + umount $devices || true - # Determine if we have a RAID set - if [ ! -z "$raid_device_name" ]; then - echo "Creating RAID set on '$raid_device_name' for devices '$devices'" - device_name=/dev/$raid_device_name - /sbin/udevadm control --stop-exec-queue - yes | /sbin/mdadm --create $device_name --level=0 -c256 --raid-devices=$num_devices $devices - /sbin/udevadm control --start-exec-queue - /sbin/mdadm --detail --scan > /etc/mdadm.conf - /sbin/blockdev --setra 32 $device_name + # Determine if we have a RAID set + if [ ! -z "$raid_device_name" ]; then + echo "Creating RAID set on '$raid_device_name' for devices '$devices'" + device_name=/dev/$raid_device_name + /sbin/udevadm control --stop-exec-queue + yes | /sbin/mdadm --create $device_name --level=0 -c256 --raid-devices=$num_devices $devices + /sbin/udevadm control --start-exec-queue + /sbin/mdadm --detail --scan >/etc/mdadm.conf + /sbin/blockdev --setra 32 $device_name + else + device_name="/dev/$device_names" + fi + + # Mount the $root_dir drive(s) + /sbin/mkfs.$fs_type $mount_options -f $device_name + # We add an entry for the device to /etc/fstab so it is automatically mounted following a + # machine reboot. The device is not guaranteed to be assigned the same name across restarts so + # we use its UUID in order to identify it. + # + # We also specify type=$fs_type in the /etc/fstab entry because specifying type=auto on + # Amazon Linux AMI 2018.03 leads to the drive not being mounted automatically following a + # machine reboot. + device_uuid=$(blkid -o value -s UUID "$device_name") + echo "Adding entry to /etc/fstab for device '$device_name' with UUID '$device_uuid'" + echo "UUID=$device_uuid /$root_dir $fs_type noatime 0 0" | tee -a /etc/fstab + mkdir /$root_dir || true + chmod 777 /$root_dir + mount -t $fs_type "UUID=$device_uuid" /$root_dir + for sub_dir in $sub_dirs; do + mkdir -p /$root_dir/$sub_dir + chmod 1777 /$root_dir/$sub_dir + done + chown -R $user_group /$root_dir else - device_name="/dev/$device_names" + echo "Unsupported OS '$(uname)'" + exit 0 fi - - # Mount the $root_dir drive(s) - /sbin/mkfs.$fs_type $mount_options -f $device_name - # We add an entry for the device to /etc/fstab so it is automatically mounted following a - # machine reboot. The device is not guaranteed to be assigned the same name across restarts so - # we use its UUID in order to identify it. - # - # We also specify type=$fs_type in the /etc/fstab entry because specifying type=auto on - # Amazon Linux AMI 2018.03 leads to the drive not being mounted automatically following a - # machine reboot. - device_uuid=$(blkid -o value -s UUID "$device_name") - echo "Adding entry to /etc/fstab for device '$device_name' with UUID '$device_uuid'" - echo "UUID=$device_uuid /$root_dir $fs_type noatime 0 0" | tee -a /etc/fstab - mkdir /$root_dir || true - chmod 777 /$root_dir - mount -t $fs_type "UUID=$device_uuid" /$root_dir - for sub_dir in $sub_dirs - do - mkdir -p /$root_dir/$sub_dir - chmod 1777 /$root_dir/$sub_dir - done - chown -R $user_group /$root_dir - else - echo "Unsupported OS '$(uname)'" - exit 0 - fi } mount_drive data "db tmp" "$data_device_names" "$data_raid_device_name" "$mount_options" "$fs_type" "$user_group" diff --git a/buildscripts/poetry_sync.sh b/buildscripts/poetry_sync.sh index ed833808ec9..975e5917ade 100755 --- a/buildscripts/poetry_sync.sh +++ b/buildscripts/poetry_sync.sh @@ -40,7 +40,7 @@ while getopts p:fin opt; do esac done -run () { +run() { echo "$@" if [[ "${dry_run}" == 1 ]]; then return @@ -79,17 +79,17 @@ if [[ "${allow_no_venv}" != 1 ]]; then fi # check if poetry should be installed via pip -need_poetry_install=0 # 0 = no, 1 = yes +need_poetry_install=0 # 0 = no, 1 = yes if ! "${py3}" -m pip show poetry &>/dev/null; then echo "Poetry not found in this interpreter, installing via pip." >&2 need_poetry_install=1 fi # we'll need to use pip this time around -if (( need_poetry_install )); then +if ((need_poetry_install)); then run "${py3}" -m pip install "${pip_opts[@]}" -r poetry_requirements.txt fi run env \ - PYTHON_KEYRING_BACKEND="keyring.backends.null.Keyring" \ - "${py3}" -m poetry sync --no-root + PYTHON_KEYRING_BACKEND="keyring.backends.null.Keyring" \ + "${py3}" -m poetry sync --no-root diff --git a/buildscripts/setup_engflow_creds.sh b/buildscripts/setup_engflow_creds.sh index 2e7850139df..e51214dcf83 100644 --- a/buildscripts/setup_engflow_creds.sh +++ b/buildscripts/setup_engflow_creds.sh @@ -10,28 +10,28 @@ ZIP_FILE=$3 LOCAL=$4 if [ -z "$REMOTE_USER" ] || [ -z "$REMOTE_HOST" ] || [ -z "$ZIP_FILE" ]; then - echo "Usage: $0 " - exit 1 + echo "Usage: $0 " + exit 1 fi if [ -z "$LOCAL" ]; then - ssh ${REMOTE_USER}@${REMOTE_HOST} "mkdir -p ~/.engflow/creds" - scp ${ZIP_FILE} ${REMOTE_USER}@${REMOTE_HOST}:~/.engflow/creds - ssh ${REMOTE_USER}@${REMOTE_HOST} "cd ~/.engflow/creds; unzip -o engflow-mTLS.zip; rm engflow-mTLS.zip" + ssh ${REMOTE_USER}@${REMOTE_HOST} "mkdir -p ~/.engflow/creds" + scp ${ZIP_FILE} ${REMOTE_USER}@${REMOTE_HOST}:~/.engflow/creds + ssh ${REMOTE_USER}@${REMOTE_HOST} "cd ~/.engflow/creds; unzip -o engflow-mTLS.zip; rm engflow-mTLS.zip" - ssh ${REMOTE_USER}@${REMOTE_HOST} "chown ${REMOTE_USER}:${REMOTE_USER} /home/${REMOTE_USER}/.engflow/creds/engflow.crt /home/${REMOTE_USER}/.engflow/creds/engflow.key" - ssh ${REMOTE_USER}@${REMOTE_HOST} "chmod 600 /home/${REMOTE_USER}/.engflow/creds/engflow.crt /home/${REMOTE_USER}/.engflow/creds/engflow.key" + ssh ${REMOTE_USER}@${REMOTE_HOST} "chown ${REMOTE_USER}:${REMOTE_USER} /home/${REMOTE_USER}/.engflow/creds/engflow.crt /home/${REMOTE_USER}/.engflow/creds/engflow.key" + ssh ${REMOTE_USER}@${REMOTE_HOST} "chmod 600 /home/${REMOTE_USER}/.engflow/creds/engflow.crt /home/${REMOTE_USER}/.engflow/creds/engflow.key" - ssh ${REMOTE_USER}@${REMOTE_HOST} "echo \"common --tls_client_certificate=/home/${REMOTE_USER}/.engflow/creds/engflow.crt\" >> ~/.bazelrc" - ssh ${REMOTE_USER}@${REMOTE_HOST} "echo \"common --tls_client_key=/home/${REMOTE_USER}/.engflow/creds/engflow.key\" >> ~/.bazelrc" + ssh ${REMOTE_USER}@${REMOTE_HOST} "echo \"common --tls_client_certificate=/home/${REMOTE_USER}/.engflow/creds/engflow.crt\" >> ~/.bazelrc" + ssh ${REMOTE_USER}@${REMOTE_HOST} "echo \"common --tls_client_key=/home/${REMOTE_USER}/.engflow/creds/engflow.key\" >> ~/.bazelrc" else - mkdir -p $HOME/.engflow/creds - unzip -o "$ZIP_FILE" - rm "$ZIP_FILE" - mv engflow.crt $HOME/.engflow/creds - mv engflow.key $HOME/.engflow/creds - chown $USER $HOME/.engflow/creds/engflow.crt $HOME/.engflow/creds/engflow.key - chmod 600 $HOME/.engflow/creds/engflow.crt $HOME/.engflow/creds/engflow.key - echo "common --tls_client_certificate=$HOME/.engflow/creds/engflow.crt" >> $HOME/.bazelrc - echo "common --tls_client_key=$HOME/.engflow/creds/engflow.key" >> $HOME/.bazelrc + mkdir -p $HOME/.engflow/creds + unzip -o "$ZIP_FILE" + rm "$ZIP_FILE" + mv engflow.crt $HOME/.engflow/creds + mv engflow.key $HOME/.engflow/creds + chown $USER $HOME/.engflow/creds/engflow.crt $HOME/.engflow/creds/engflow.key + chmod 600 $HOME/.engflow/creds/engflow.crt $HOME/.engflow/creds/engflow.key + echo "common --tls_client_certificate=$HOME/.engflow/creds/engflow.crt" >>$HOME/.bazelrc + echo "common --tls_client_key=$HOME/.engflow/creds/engflow.key" >>$HOME/.bazelrc fi diff --git a/buildscripts/shellscripts-linters.sh b/buildscripts/shellscripts-linters.sh deleted file mode 100755 index f4f2bf69cc1..00000000000 --- a/buildscripts/shellscripts-linters.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set +o errexit - -shfmt=shfmt -if [ -n "$SHFMT_PATH" ]; then - shfmt=$(readlink $SHFMT_PATH) -fi - -if [ -n "$BUILD_WORKSPACE_DIRECTORY" ]; then - cd $BUILD_WORKSPACE_DIRECTORY -fi - -if ! command -v $shfmt &>/dev/null; then - echo "Could not find shfmt at $shfmt" - exit 1 -fi - -lint_dirs="evergreen" - -if [ "$1" = "fix" ]; then - $shfmt -w -i 2 -bn -sr "$lint_dirs" -fi - -output_file="shfmt_output.txt" -exit_code=0 - -$shfmt -d -i 2 -bn -sr "$lint_dirs" >"$output_file" -if [ -s "$output_file" ]; then - echo "ERROR: Found formatting errors in shell script files in directories: $lint_dirs" - echo "" - cat "$output_file" - echo "" - echo "To fix formatting errors run" - echo "" - echo " ./buildscripts/shellscripts-linters.sh fix" - echo "" - exit_code=1 -fi -rm -rf "$output_file" - -exit "$exit_code" diff --git a/buildscripts/tests/sbom_linter/inputs/import_script_with_mongo_prefix_version.sh b/buildscripts/tests/sbom_linter/inputs/import_script_with_mongo_prefix_version.sh index 80ca5ee519d..e66b78742be 100755 --- a/buildscripts/tests/sbom_linter/inputs/import_script_with_mongo_prefix_version.sh +++ b/buildscripts/tests/sbom_linter/inputs/import_script_with_mongo_prefix_version.sh @@ -8,4 +8,3 @@ set -vx NAME=protobuf VERSION="mongo/v4.25.0" - diff --git a/buildscripts/tests/sbom_linter/inputs/kafka_missing_version_import.sh b/buildscripts/tests/sbom_linter/inputs/kafka_missing_version_import.sh index 789108093cb..2a8a2dff906 100644 --- a/buildscripts/tests/sbom_linter/inputs/kafka_missing_version_import.sh +++ b/buildscripts/tests/sbom_linter/inputs/kafka_missing_version_import.sh @@ -7,8 +7,8 @@ set -euo pipefail IFS=$'\n\t' if [ "$#" -ne 0 ]; then - echo "This script does not take any arguments" - exit 1 + echo "This script does not take any arguments" + exit 1 fi # Create a temporary directory to clone and configure librdkafka @@ -42,11 +42,11 @@ mv config.h $PLATFORM_DIR/$platformName/include # Remove un-used files rm -rf CHANGELOG.md CODE_OF_CONDUCT.md CONFIGURATION.md CONTRIBUTING.md INTRODUCTION.md \ - README.md README.win32 STATISTICS.md config.log.old dev-conf.sh examples/ \ - CMakeLists.txt lds-gen.py mklove/ packaging/ service.yml tests/ vcpkg.json win32/ \ - Makefile Makefile.config config.cache configure.self configure debian mainpage.doxy Doxyfile \ - src/CMakeLists.txt src/Makefile src/generate_proto.sh src/librdkafka_cgrp_synch.png src/statistics_schema.json \ - src-cpp/CMakeLists.txt src-cpp/Makefile src-cpp/README.md config.log + README.md README.win32 STATISTICS.md config.log.old dev-conf.sh examples/ \ + CMakeLists.txt lds-gen.py mklove/ packaging/ service.yml tests/ vcpkg.json win32/ \ + Makefile Makefile.config config.cache configure.self configure debian mainpage.doxy Doxyfile \ + src/CMakeLists.txt src/Makefile src/generate_proto.sh src/librdkafka_cgrp_synch.png src/statistics_schema.json \ + src-cpp/CMakeLists.txt src-cpp/Makefile src-cpp/README.md config.log pushd src # Replace all instances of the string "LZ4" and "XXH" with "KLZ4" and "KXXH" in the C source code. diff --git a/buildscripts/tests/sbom_linter/inputs/kafka_valid_import.sh b/buildscripts/tests/sbom_linter/inputs/kafka_valid_import.sh index 5ad9d5aac8f..1ce4bb10406 100755 --- a/buildscripts/tests/sbom_linter/inputs/kafka_valid_import.sh +++ b/buildscripts/tests/sbom_linter/inputs/kafka_valid_import.sh @@ -7,8 +7,8 @@ set -euo pipefail IFS=$'\n\t' if [ "$#" -ne 0 ]; then - echo "This script does not take any arguments" - exit 1 + echo "This script does not take any arguments" + exit 1 fi # Create a temporary directory to clone and configure librdkafka @@ -43,11 +43,11 @@ mv config.h $PLATFORM_DIR/$platformName/include # Remove un-used files rm -rf CHANGELOG.md CODE_OF_CONDUCT.md CONFIGURATION.md CONTRIBUTING.md INTRODUCTION.md \ - README.md README.win32 STATISTICS.md config.log.old dev-conf.sh examples/ \ - CMakeLists.txt lds-gen.py mklove/ packaging/ service.yml tests/ vcpkg.json win32/ \ - Makefile Makefile.config config.cache configure.self configure debian mainpage.doxy Doxyfile \ - src/CMakeLists.txt src/Makefile src/generate_proto.sh src/librdkafka_cgrp_synch.png src/statistics_schema.json \ - src-cpp/CMakeLists.txt src-cpp/Makefile src-cpp/README.md config.log + README.md README.win32 STATISTICS.md config.log.old dev-conf.sh examples/ \ + CMakeLists.txt lds-gen.py mklove/ packaging/ service.yml tests/ vcpkg.json win32/ \ + Makefile Makefile.config config.cache configure.self configure debian mainpage.doxy Doxyfile \ + src/CMakeLists.txt src/Makefile src/generate_proto.sh src/librdkafka_cgrp_synch.png src/statistics_schema.json \ + src-cpp/CMakeLists.txt src-cpp/Makefile src-cpp/README.md config.log pushd src # Replace all instances of the string "LZ4" and "XXH" with "KLZ4" and "KXXH" in the C source code. diff --git a/buildscripts/tests/sbom_linter/inputs/kafka_wrong_version_import.sh b/buildscripts/tests/sbom_linter/inputs/kafka_wrong_version_import.sh index 83a7fc9c8ac..e596fa96b4b 100644 --- a/buildscripts/tests/sbom_linter/inputs/kafka_wrong_version_import.sh +++ b/buildscripts/tests/sbom_linter/inputs/kafka_wrong_version_import.sh @@ -7,8 +7,8 @@ set -euo pipefail IFS=$'\n\t' if [ "$#" -ne 0 ]; then - echo "This script does not take any arguments" - exit 1 + echo "This script does not take any arguments" + exit 1 fi # Create a temporary directory to clone and configure librdkafka @@ -43,11 +43,11 @@ mv config.h $PLATFORM_DIR/$platformName/include # Remove un-used files rm -rf CHANGELOG.md CODE_OF_CONDUCT.md CONFIGURATION.md CONTRIBUTING.md INTRODUCTION.md \ - README.md README.win32 STATISTICS.md config.log.old dev-conf.sh examples/ \ - CMakeLists.txt lds-gen.py mklove/ packaging/ service.yml tests/ vcpkg.json win32/ \ - Makefile Makefile.config config.cache configure.self configure debian mainpage.doxy Doxyfile \ - src/CMakeLists.txt src/Makefile src/generate_proto.sh src/librdkafka_cgrp_synch.png src/statistics_schema.json \ - src-cpp/CMakeLists.txt src-cpp/Makefile src-cpp/README.md config.log + README.md README.win32 STATISTICS.md config.log.old dev-conf.sh examples/ \ + CMakeLists.txt lds-gen.py mklove/ packaging/ service.yml tests/ vcpkg.json win32/ \ + Makefile Makefile.config config.cache configure.self configure debian mainpage.doxy Doxyfile \ + src/CMakeLists.txt src/Makefile src/generate_proto.sh src/librdkafka_cgrp_synch.png src/statistics_schema.json \ + src-cpp/CMakeLists.txt src-cpp/Makefile src-cpp/README.md config.log pushd src # Replace all instances of the string "LZ4" and "XXH" with "KLZ4" and "KXXH" in the C source code. diff --git a/buildscripts/yamllinters.sh b/buildscripts/yamllinters.sh index d7229979bc6..2dc4e48b67c 100755 --- a/buildscripts/yamllinters.sh +++ b/buildscripts/yamllinters.sh @@ -5,14 +5,14 @@ cd "$BASEDIR/../" yamllint -c etc/yamllint_config.yml buildscripts etc jstests -PATH="$PATH:$HOME" evergreen evaluate etc/evergreen.yml > etc/evaluated_evergreen.yml -PATH="$PATH:$HOME" evergreen evaluate etc/evergreen_nightly.yml > etc/evaluated_evergreen_nightly.yml +PATH="$PATH:$HOME" evergreen evaluate etc/evergreen.yml >etc/evaluated_evergreen.yml +PATH="$PATH:$HOME" evergreen evaluate etc/evergreen_nightly.yml >etc/evaluated_evergreen_nightly.yml # Remove references to the DSI repo before evergreen evaluate. # The DSI module references break 'evaluate', the system_perf config should # parse without them, and we don't want changes to the DSI repository to # break checking that the rest of the imports etc. work. -awk '/lint_yaml trim start/{drop=1} /lint_yaml trim end/{drop=0} !drop' etc/system_perf.yml > etc/trimmed_system_perf.yml -PATH="$PATH:$HOME" evergreen evaluate etc/trimmed_system_perf.yml > etc/evaluated_system_perf.yml +awk '/lint_yaml trim start/{drop=1} /lint_yaml trim end/{drop=0} !drop' etc/system_perf.yml >etc/trimmed_system_perf.yml +PATH="$PATH:$HOME" evergreen evaluate etc/trimmed_system_perf.yml >etc/evaluated_system_perf.yml python -m evergreen_lint -c ./etc/evergreen_lint.yml lint diff --git a/etc/set_up_workstation.sh b/etc/set_up_workstation.sh index 8459b367fe7..5f34dca2977 100755 --- a/etc/set_up_workstation.sh +++ b/etc/set_up_workstation.sh @@ -1,5 +1,5 @@ silent_grep() { - command grep -q > /dev/null 2>&1 "$@" + command grep -q "$@" >/dev/null 2>&1 } idem_file_append() { @@ -19,10 +19,10 @@ idem_file_append() { local end_marker="# END $2" if ! silent_grep "^$start_marker" "$1"; then { - echo -e "\n$start_marker"; - echo -e "$3"; - echo -e "$end_marker"; - } >> "$1" + echo -e "\n$start_marker" + echo -e "$3" + echo -e "$end_marker" + } >>"$1" fi } @@ -30,7 +30,8 @@ setup_bash() { # Bash profile should source .bashrc echo "################################################################################" echo "Setting up bash..." - local block=$(cat < /dev/null; then + if command -v poetry &>/dev/null; then echo "'poetry' command exists; skipping setup" else pipx install poetry --pip-args="-r $(pwd)/poetry_requirements.txt" @@ -78,7 +79,7 @@ setup_poetry() { setup_pipx() { echo "################################################################################" echo "Installing 'pipx' command..." - if command -v pipx &> /dev/null; then + if command -v pipx &>/dev/null; then echo "'pipx' command exists; skipping setup" else export PATH="$PATH:$HOME/.local/bin" @@ -112,7 +113,7 @@ setup_db_contrib_tool() { echo "################################################################################" echo "Installing 'db-contrib-tool' command..." export PATH="$PATH:$HOME/.local/bin" - if command -v db-contrib-tool &> /dev/null; then + if command -v db-contrib-tool &>/dev/null; then echo "'db-contrib-tool' command exists; skipping setup" else pipx install db-contrib-tool @@ -125,7 +126,7 @@ setup_clang_config() { echo "Installing clang config..." bazel build compiledb - + echo "Finished installing clang config..." } @@ -155,14 +156,14 @@ run_setup() { set +o nounset source ~/.bashrc set -o nounset - + setup_bash setup_clang_config setup_gdb setup_pipx - setup_db_contrib_tool # This step requires `setup_pipx` to have been run. - setup_poetry # This step requires `setup_pipx` to have been run. + setup_db_contrib_tool # This step requires `setup_pipx` to have been run. + setup_poetry # This step requires `setup_pipx` to have been run. setup_mongo_venv # This step requires `setup_poetry` to have been run. diff --git a/evergreen/BUILD.bazel b/evergreen/BUILD.bazel index 0cdc35b5993..b8bff72ee06 100644 --- a/evergreen/BUILD.bazel +++ b/evergreen/BUILD.bazel @@ -296,11 +296,6 @@ sh_binary( srcs = ["lint_fuzzer_sanity_patch.sh"], ) -sh_binary( - name = "lint_shellscripts", - srcs = ["lint_shellscripts.sh"], -) - sh_binary( name = "lint_yaml", srcs = ["lint_yaml.sh"], diff --git a/evergreen/antithesis_image_build_and_push.sh b/evergreen/antithesis_image_build_and_push.sh index b56d6b29662..dc66f96a685 100644 --- a/evergreen/antithesis_image_build_and_push.sh +++ b/evergreen/antithesis_image_build_and_push.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" set -o errexit @@ -8,19 +8,19 @@ antithesis_repo="us-central1-docker.pkg.dev/molten-verve-216720/mongodb-reposito # tag images as evergreen[-${antithesis_build_type}]-{latest,patch} or just ${antithesis_image_tag} if [ -n "${antithesis_image_tag:-}" ]; then - echo "Using provided tag: '$antithesis_image_tag' for docker pushes" - tag=$antithesis_image_tag + echo "Using provided tag: '$antithesis_image_tag' for docker pushes" + tag=$antithesis_image_tag else - tag="evergreen" - if [[ -n "${antithesis_build_type}" ]]; then - tag="${tag}-${antithesis_build_type}" - fi + tag="evergreen" + if [[ -n "${antithesis_build_type}" ]]; then + tag="${tag}-${antithesis_build_type}" + fi - if [ "${is_patch}" = "true" ]; then - tag="${tag}-patch" - else - tag="${tag}-latest-${branch_name}" - fi + if [ "${is_patch}" = "true" ]; then + tag="${tag}-patch" + else + tag="${tag}-latest-${branch_name}" + fi fi # Clean up any leftover docker artifacts @@ -33,9 +33,9 @@ sudo docker network prune --force sudo service docker stop sudo mkdir -p /data/mci/docker if ! sudo jq -e . /etc/docker/daemon.json; then - echo "docker daemon.json did not exist or was invalid" - echo "setting docker daemon.json to {}" - sudo sh -c 'echo "{}" > /etc/docker/daemon.json' + echo "docker daemon.json did not exist or was invalid" + echo "setting docker daemon.json to {}" + sudo sh -c 'echo "{}" > /etc/docker/daemon.json' fi MODIFIED_JSON=$(sudo jq '."data-root" |= "/data/mci/docker"' /etc/docker/daemon.json) sudo echo "${MODIFIED_JSON}" | sudo tee /etc/docker/daemon.json @@ -43,7 +43,7 @@ echo "docker daemon.json: set data-root to /data/mci/docker" sudo service docker start # Login -echo "${antithesis_repo_key}" > mongodb.key.json +echo "${antithesis_repo_key}" >mongodb.key.json cat mongodb.key.json | sudo docker login -u _json_key https://us-central1-docker.pkg.dev --password-stdin rm mongodb.key.json @@ -64,15 +64,15 @@ timeout -v 1800 docker exec workload buildscripts/resmoke.py run --suite ${suite RET=$? set -o errexit -docker-compose -f docker_compose/${suite}/docker-compose.yml logs > docker_logs.txt +docker-compose -f docker_compose/${suite}/docker-compose.yml logs >docker_logs.txt docker-compose -f docker_compose/${suite}/docker-compose.yml down # Change the permissions of all of the files in the docker compose directory to the current user. # Some of the data files cannot be archived otherwise. sudo chown -R $USER docker_compose/${suite}/ if [ $RET -ne 0 ]; then - echo "Resmoke sanity check has failed" - exit $RET + echo "Resmoke sanity check has failed" + exit $RET fi # Push Image diff --git a/evergreen/bazel.sh b/evergreen/bazel.sh index f0d6fc2b774..c9761f726b9 100755 --- a/evergreen/bazel.sh +++ b/evergreen/bazel.sh @@ -1,7 +1,7 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" source "$DIR/bazel_utility_functions.sh" ( - cd $DIR/.. - exec $(bazel_get_binary_path) "$@" + cd $DIR/.. + exec $(bazel_get_binary_path) "$@" ) diff --git a/evergreen/bazel_RBE_supported.sh b/evergreen/bazel_RBE_supported.sh index c074be50854..54fe9d9512a 100644 --- a/evergreen/bazel_RBE_supported.sh +++ b/evergreen/bazel_RBE_supported.sh @@ -1,11 +1,11 @@ bazel_rbe_supported() { - OS="$(uname)" - ARCH="$(uname -m)" + OS="$(uname)" + ARCH="$(uname -m)" - if [ "$ARCH" == "aarch64" ] || [ "$ARCH" == "arm64" ] || [ "$ARCH" == "x86_64" ]; then - return 0 - else - return 1 - fi + if [ "$ARCH" == "aarch64" ] || [ "$ARCH" == "arm64" ] || [ "$ARCH" == "x86_64" ]; then + return 0 + else + return 1 + fi } diff --git a/evergreen/bazel_compile.sh b/evergreen/bazel_compile.sh index d9c3f918f81..6f67070e27a 100755 --- a/evergreen/bazel_compile.sh +++ b/evergreen/bazel_compile.sh @@ -8,7 +8,7 @@ # * ${args} - List of additional Bazel arguments (e.g.: "--config=clang-tidy") # Needed for evergreen scripts that use evergreen expansions and utility methods. -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -22,67 +22,67 @@ activate_venv # evergreen patch. build_patch_id="${build_patch_id:-${reuse_compile_from}}" if [ -n "${build_patch_id}" ]; then - echo "build_patch_id detected, trying to skip task" + echo "build_patch_id detected, trying to skip task" - # On windows we change the extension to zip - if [ -z "${ext}" ]; then - ext="tgz" - fi + # On windows we change the extension to zip + if [ -z "${ext}" ]; then + ext="tgz" + fi - extra_db_contrib_args="" + extra_db_contrib_args="" - # get the platform of the dist archive. This is needed if - # db-contrib-tool cannot autodetect the platform of the ec2 instance. - regex='MONGO_DISTMOD=([a-z0-9]*)' - if [[ ${bazel_compile_flags} =~ ${regex} ]]; then - extra_db_contrib_args="${extra_db_contrib_args} --platform=${BASH_REMATCH[1]}" - fi + # get the platform of the dist archive. This is needed if + # db-contrib-tool cannot autodetect the platform of the ec2 instance. + regex='MONGO_DISTMOD=([a-z0-9]*)' + if [[ ${bazel_compile_flags} =~ ${regex} ]]; then + extra_db_contrib_args="${extra_db_contrib_args} --platform=${BASH_REMATCH[1]}" + fi - download_dir="./tmp_db_contrib_tool_download_dir" - rm -rf ${download_dir} + download_dir="./tmp_db_contrib_tool_download_dir" + rm -rf ${download_dir} - if [ "${task_name}" = "archive_dist_test" ]; then - file_name="dist-test-stripped.${ext}" - invocation="db-contrib-tool setup-repro-env ${build_patch_id} \ + if [ "${task_name}" = "archive_dist_test" ]; then + file_name="dist-test-stripped.${ext}" + invocation="db-contrib-tool setup-repro-env ${build_patch_id} \ --variant=${compile_variant} --extractDownloads=False \ --binariesName=${file_name} --installDir=${download_dir} ${extra_db_contrib_args}" - fi + fi - if [ "${task_name}" = "archive_dist_test_debug" ]; then - file_name="dist-test-debug.${ext}" - invocation="db-contrib-tool setup-repro-env ${build_patch_id} \ + if [ "${task_name}" = "archive_dist_test_debug" ]; then + file_name="dist-test-debug.${ext}" + invocation="db-contrib-tool setup-repro-env ${build_patch_id} \ --variant=${compile_variant} --extractDownloads=False \ --debugsymbolsName=${file_name} --installDir=${download_dir} \ --skipBinaries --downloadSymbols ${extra_db_contrib_args}" - fi - - if [ -n "${invocation}" ]; then - setup_db_contrib_tool - - echo "db-contrib-tool invocation: ${invocation}" - eval ${invocation} - if [ $? -ne 0 ]; then - echo "Could not retrieve files with db-contrib-tool" - exit 1 fi - file_location=$(find "${download_dir}" -name "${file_name}") - echo "Downloaded: ${file_location}" - mkdir -p bazel-bin - mv "${file_location}" "bazel-bin/${file_name}" - echo "Moved ${file_name} to the correct location" - echo "Skipping ${task_name} compile" - exit 0 - fi - echo "Could not skip ${task_name} compile, compiling as normal" + if [ -n "${invocation}" ]; then + setup_db_contrib_tool + + echo "db-contrib-tool invocation: ${invocation}" + eval ${invocation} + if [ $? -ne 0 ]; then + echo "Could not retrieve files with db-contrib-tool" + exit 1 + fi + file_location=$(find "${download_dir}" -name "${file_name}") + echo "Downloaded: ${file_location}" + mkdir -p bazel-bin + mv "${file_location}" "bazel-bin/${file_name}" + echo "Moved ${file_name} to the correct location" + echo "Skipping ${task_name} compile" + exit 0 + fi + + echo "Could not skip ${task_name} compile, compiling as normal" fi # --build-mongot is a compile flag used by the evergreen build variants that run end-to-end search # suites, as it downloads the necessary mongot binary. if [ "${build_mongot}" = "true" ]; then - setup_db_contrib_tool - use_db_contrib_tool_mongot - bazel_args="${bazel_args} --include_mongot=True" + setup_db_contrib_tool + use_db_contrib_tool_mongot + bazel_args="${bazel_args} --include_mongot=True" fi # This is hacky way to pass off build time from archive_dist_test to archive_dist_test_debug @@ -91,14 +91,14 @@ fi # build-id for debugging as they will be different when -Wl,-S is passed in. # The relinked binaries should still be hash identical when stripped with strip if [ "${skip_debug_link}" = "true" ]; then - export compile_variant="${compile_variant}" - export version_id="${version_id}" - if [ "${task_name}" = "archive_dist_test" ]; then - task_compile_flags="${task_compile_flags} --simple_build_id=True --linkopt='-Wl,-S' --separate_debug=False" - fi - if [ "${task_name}" = "archive_dist_test_debug" ]; then - task_compile_flags="${task_compile_flags} --simple_build_id=True" - fi + export compile_variant="${compile_variant}" + export version_id="${version_id}" + if [ "${task_name}" = "archive_dist_test" ]; then + task_compile_flags="${task_compile_flags} --simple_build_id=True --linkopt='-Wl,-S' --separate_debug=False" + fi + if [ "${task_name}" = "archive_dist_test_debug" ]; then + task_compile_flags="${task_compile_flags} --simple_build_id=True" + fi fi set -o pipefail @@ -110,7 +110,7 @@ source ./evergreen/bazel_utility_functions.sh source ./evergreen/bazel_RBE_supported.sh if [[ "${evergreen_remote_exec}" != "on" ]]; then - LOCAL_ARG="$LOCAL_ARG --jobs=auto" + LOCAL_ARG="$LOCAL_ARG --jobs=auto" fi BAZEL_BINARY=$(bazel_get_binary_path) @@ -119,19 +119,19 @@ BAZEL_BINARY=$(bazel_get_binary_path) # for retries. TIMEOUT_CMD="" if [ -n "${build_timeout_seconds}" ]; then - TIMEOUT_CMD="timeout ${build_timeout_seconds}" + TIMEOUT_CMD="timeout ${build_timeout_seconds}" elif [[ "${evergreen_remote_exec}" == "on" ]]; then - # Timeout remote execution runs in 60 minutes as a workaround for - # scheduling timeout bugs - TIMEOUT_CMD="timeout 3600" + # Timeout remote execution runs in 60 minutes as a workaround for + # scheduling timeout bugs + TIMEOUT_CMD="timeout 3600" fi if is_ppc64le; then - LOCAL_ARG="$LOCAL_ARG --jobs=48" + LOCAL_ARG="$LOCAL_ARG --jobs=48" fi if is_s390x; then - LOCAL_ARG="$LOCAL_ARG --jobs=16" + LOCAL_ARG="$LOCAL_ARG --jobs=16" fi # If we are doing a patch build or we are building a non-push @@ -139,23 +139,23 @@ fi # flag. Otherwise, this is potentially a build that "leaves # the building", so we do want that flag. if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then - echo "This is a non-release build." + echo "This is a non-release build." else - LOCAL_ARG="$LOCAL_ARG --config=public-release" + LOCAL_ARG="$LOCAL_ARG --config=public-release" fi for i in {1..3}; do - eval ${TIMEOUT_CMD} $BAZEL_BINARY build --verbose_failures $LOCAL_ARG ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \ - --define=MONGO_VERSION=${version} ${patch_compile_flags} ${targets} 2>&1 | tee bazel_stdout.log \ - && RET=0 && break || RET=$? && sleep 60 - if [ $RET -eq 124 ]; then - echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..." - else - echo "Errors were found during the bazel run, here are the errors:" 1>&2 - grep "ERROR:" bazel_stdout.log 1>&2 - echo "Bazel failed to execute, retrying..." - fi - $BAZEL_BINARY shutdown + eval ${TIMEOUT_CMD} $BAZEL_BINARY build --verbose_failures $LOCAL_ARG ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \ + --define=MONGO_VERSION=${version} ${patch_compile_flags} ${targets} 2>&1 | tee bazel_stdout.log && + RET=0 && break || RET=$? && sleep 60 + if [ $RET -eq 124 ]; then + echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..." + else + echo "Errors were found during the bazel run, here are the errors:" 1>&2 + grep "ERROR:" bazel_stdout.log 1>&2 + echo "Bazel failed to execute, retrying..." + fi + $BAZEL_BINARY shutdown done exit $RET diff --git a/evergreen/bazel_coverage.sh b/evergreen/bazel_coverage.sh index f65038c9a17..40fc600faf5 100644 --- a/evergreen/bazel_coverage.sh +++ b/evergreen/bazel_coverage.sh @@ -8,7 +8,7 @@ # * ${args} - Extra command line args to pass to "bazel coverage" # Needed for evergreen scripts that use evergreen expansions and utility methods. -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -24,7 +24,7 @@ eval echo "Execution environment: Args: ${args} Target: ${target}" BAZEL_BINARY=bazel # Print command being run to file that can be uploaded -echo "python buildscripts/install_bazel.py" > bazel-invocation.txt +echo "python buildscripts/install_bazel.py" >bazel-invocation.txt -echo " bazel coverage ${args} ${target}" >> bazel-invocation.txt +echo " bazel coverage ${args} ${target}" >>bazel-invocation.txt $BAZEL_BINARY coverage ${args} ${target} diff --git a/evergreen/bazel_run.sh b/evergreen/bazel_run.sh index 87647391d6a..46b24285f8b 100644 --- a/evergreen/bazel_run.sh +++ b/evergreen/bazel_run.sh @@ -8,7 +8,7 @@ # * ${redact_args} - If set, redact the args in the report # Needed for evergreen scripts that use evergreen expansions and utility methods. -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -25,45 +25,45 @@ source ./evergreen/bazel_utility_functions.sh source ./evergreen/bazel_RBE_supported.sh if bazel_rbe_supported; then - LOCAL_ARG="" + LOCAL_ARG="" else - LOCAL_ARG="--config=local" + LOCAL_ARG="--config=local" fi if [[ "${evergreen_remote_exec}" != "on" ]]; then - LOCAL_ARG="--config=local" + LOCAL_ARG="--config=local" fi BAZEL_BINARY=$(bazel_get_binary_path) # AL2 stores certs in a nonstandard location if [[ -f /etc/os-release ]]; then - DISTRO=$(awk -F '[="]*' '/^PRETTY_NAME/ { print $2 }' < /etc/os-release) - if [[ $DISTRO == "Amazon Linux 2" ]]; then - export SSL_CERT_DIR=/etc/pki/tls/certs - export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt - elif [[ $DISTRO == "Red Hat Enterprise Linux"* ]]; then - export SSL_CERT_DIR=/etc/pki/ca-trust/extracted/pem - export SSL_CERT_FILE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem - fi + DISTRO=$(awk -F '[="]*' '/^PRETTY_NAME/ { print $2 }' bazel-invocation.txt -echo "bazel run --verbose_failures ${bazel_compile_flags} ${task_compile_flags} ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >> bazel-invocation.txt +echo "python buildscripts/install_bazel.py" >bazel-invocation.txt +echo "bazel run --verbose_failures ${bazel_compile_flags} ${task_compile_flags} ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >>bazel-invocation.txt # Run bazel command, retrying up to five times MAX_ATTEMPTS=5 for ((i = 1; i <= $MAX_ATTEMPTS; i++)); do - eval $env $BAZEL_BINARY run --verbose_failures $LOCAL_ARG ${target} ${args} >> bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 10 - if [ $i -lt $MAX_ATTEMPTS ]; then echo "Bazel failed to execute, retrying ($(($i + 1)) of $MAX_ATTEMPTS attempts)... " >> bazel_output.log 2>&1; fi - $BAZEL_BINARY shutdown + eval $env $BAZEL_BINARY run --verbose_failures $LOCAL_ARG ${target} ${args} >>bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 10 + if [ $i -lt $MAX_ATTEMPTS ]; then echo "Bazel failed to execute, retrying ($(($i + 1)) of $MAX_ATTEMPTS attempts)... " >>bazel_output.log 2>&1; fi + $BAZEL_BINARY shutdown done $python ./buildscripts/simple_report.py --test-name "bazel run ${INVOCATION_WITH_REDACTION}" --log-file bazel_output.log --exit-code $RET diff --git a/evergreen/bazel_test.sh b/evergreen/bazel_test.sh index 0dc300a6836..12c09a19c0c 100644 --- a/evergreen/bazel_test.sh +++ b/evergreen/bazel_test.sh @@ -5,7 +5,7 @@ # * ${targets} - Test targets # * ${bazel_args} - Extra command line args to pass to "bazel test" -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -25,7 +25,7 @@ source ./evergreen/bazel_RBE_supported.sh LOCAL_ARG="" if [[ "${evergreen_remote_exec}" != "on" ]]; then - LOCAL_ARG="$LOCAL_ARG --jobs=auto" + LOCAL_ARG="$LOCAL_ARG --jobs=auto" fi BAZEL_BINARY=$(bazel_get_binary_path) @@ -34,15 +34,15 @@ BAZEL_BINARY=$(bazel_get_binary_path) # for retries. TIMEOUT_CMD="" if [ -n "${build_timeout_seconds}" ]; then - TIMEOUT_CMD="timeout ${build_timeout_seconds}" + TIMEOUT_CMD="timeout ${build_timeout_seconds}" fi if is_ppc64le; then - LOCAL_ARG="$LOCAL_ARG --jobs=48" + LOCAL_ARG="$LOCAL_ARG --jobs=48" fi if is_s390x; then - LOCAL_ARG="$LOCAL_ARG --jobs=16" + LOCAL_ARG="$LOCAL_ARG --jobs=16" fi # If we are doing a patch build or we are building a non-push @@ -50,55 +50,55 @@ fi # flag. Otherwise, this is potentially a build that "leaves # the building", so we do want that flag. if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then - echo "This is a non-release build." + echo "This is a non-release build." else - LOCAL_ARG="$LOCAL_ARG --config=public-release" + LOCAL_ARG="$LOCAL_ARG --config=public-release" fi if [ -n "${test_timeout_sec}" ]; then - # s390x and ppc64le often run slower than other architectures - if is_s390x_or_ppc64le; then - test_timeout_sec=$(($test_timeout_sec * 4)) - fi - bazel_args="${bazel_args} --test_timeout=${test_timeout_sec}" + # s390x and ppc64le often run slower than other architectures + if is_s390x_or_ppc64le; then + test_timeout_sec=$(($test_timeout_sec * 4)) + fi + bazel_args="${bazel_args} --test_timeout=${test_timeout_sec}" fi ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} --define=MONGO_VERSION=${version} ${patch_compile_flags}" -echo ${ALL_FLAGS} > .bazel_build_flags +echo ${ALL_FLAGS} >.bazel_build_flags set +o errexit # Retry the build since it's deterministic and may fail due to transient issues. for i in {1..3}; do - eval ${TIMEOUT_CMD} ${BAZEL_BINARY} build ${ALL_FLAGS} ${targets} && RET=0 && break || RET=$? && sleep 1 - if [ $RET -eq 124 ]; then - echo "Bazel build timed out after ${build_timeout_seconds} seconds, retrying..." - else - echo "Bazel build failed, retrying..." - fi - $BAZEL_BINARY shutdown + eval ${TIMEOUT_CMD} ${BAZEL_BINARY} build ${ALL_FLAGS} ${targets} && RET=0 && break || RET=$? && sleep 1 + if [ $RET -eq 124 ]; then + echo "Bazel build timed out after ${build_timeout_seconds} seconds, retrying..." + else + echo "Bazel build failed, retrying..." + fi + $BAZEL_BINARY shutdown done for i in {1..3}; do - eval ${TIMEOUT_CMD} ${BAZEL_BINARY} test ${ALL_FLAGS} ${targets} 2>&1 | tee bazel_stdout.log \ - && RET=0 && break || RET=$? && sleep 1 - if [ $RET -eq 124 ]; then - echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..." - else - echo "Errors were found during the bazel test, failing the execution" - break - fi - $BAZEL_BINARY shutdown + eval ${TIMEOUT_CMD} ${BAZEL_BINARY} test ${ALL_FLAGS} ${targets} 2>&1 | tee bazel_stdout.log && + RET=0 && break || RET=$? && sleep 1 + if [ $RET -eq 124 ]; then + echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..." + else + echo "Errors were found during the bazel test, failing the execution" + break + fi + $BAZEL_BINARY shutdown done set -o errexit if [[ $RET != 0 ]]; then - # The --config flag needs to stay consistent between invocations to avoid evicting the previous results. - # Strip out anything that isn't a --config flag that could interfere with the run command. - CONFIG_FLAGS=$(echo "${ALL_FLAGS}" | tr ' ' '\n' | grep -- '--config' | tr '\n' ' ') + # The --config flag needs to stay consistent between invocations to avoid evicting the previous results. + # Strip out anything that isn't a --config flag that could interfere with the run command. + CONFIG_FLAGS=$(echo "${ALL_FLAGS}" | tr ' ' '\n' | grep -- '--config' | tr '\n' ' ') - eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:gather_failed_unittests + eval ${BAZEL_BINARY} run ${CONFIG_FLAGS} //buildscripts:gather_failed_unittests fi exit $RET diff --git a/evergreen/bazel_utility_functions.sh b/evergreen/bazel_utility_functions.sh index dbe470c54cd..3b8b6388811 100755 --- a/evergreen/bazel_utility_functions.sh +++ b/evergreen/bazel_utility_functions.sh @@ -1,39 +1,39 @@ is_ppc64le() { - ARCH="$(uname -m)" + ARCH="$(uname -m)" - if [[ "$ARCH" == "ppc64le" || "$ARCH" == "ppc64" || "$ARCH" == "ppc" ]]; then - return 0 - else - return 1 - fi + if [[ "$ARCH" == "ppc64le" || "$ARCH" == "ppc64" || "$ARCH" == "ppc" ]]; then + return 0 + else + return 1 + fi } is_s390x() { - ARCH="$(uname -m)" + ARCH="$(uname -m)" - if [[ "$ARCH" == "s390x" || "$ARCH" == "s390" ]]; then - return 0 - else - return 1 - fi + if [[ "$ARCH" == "s390x" || "$ARCH" == "s390" ]]; then + return 0 + else + return 1 + fi } is_s390x_or_ppc64le() { - if is_ppc64le || is_s390x; then - return 0 - else - return 1 - fi + if is_ppc64le || is_s390x; then + return 0 + else + return 1 + fi } bazel_get_binary_path() { - if is_s390x_or_ppc64le; then - echo "bazel/bazelisk.py" - elif grep -q "ID=debian" /etc/os-release; then - echo "bazel/bazelisk.py" - elif grep -q 'ID="sles"' /etc/os-release; then - echo "bazel/bazelisk.py" - else - echo "bazel" - fi + if is_s390x_or_ppc64le; then + echo "bazel/bazelisk.py" + elif grep -q "ID=debian" /etc/os-release; then + echo "bazel/bazelisk.py" + elif grep -q 'ID="sles"' /etc/os-release; then + echo "bazel/bazelisk.py" + else + echo "bazel" + fi } diff --git a/evergreen/check_idl_compat.sh b/evergreen/check_idl_compat.sh index 5566cf23dbd..99638335338 100755 --- a/evergreen/check_idl_compat.sh +++ b/evergreen/check_idl_compat.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -11,21 +11,21 @@ $python buildscripts/idl/check_stable_api_commands_have_idl_definitions.py -v -- $python buildscripts/idl/checkout_idl_files_from_past_releases.py -v idls function run_idl_check_compatibility { - dir=$1 - output=$( - python buildscripts/idl/idl_check_compatibility.py -v \ - --old-include "$dir/src" \ - --old-include "$dir/src/mongo/db/modules/enterprise/src" \ - --new-include src \ - --new-include src/mongo/db/modules/enterprise/src \ - "$dir/src" src - ) - exit_code=$? - echo "Performing idl check compatibility with release: $dir:" - echo "$output" - if [ $exit_code -ne 0 ]; then - exit 255 - fi + dir=$1 + output=$( + python buildscripts/idl/idl_check_compatibility.py -v \ + --old-include "$dir/src" \ + --old-include "$dir/src/mongo/db/modules/enterprise/src" \ + --new-include src \ + --new-include src/mongo/db/modules/enterprise/src \ + "$dir/src" src + ) + exit_code=$? + echo "Performing idl check compatibility with release: $dir:" + echo "$output" + if [ $exit_code -ne 0 ]; then + exit 255 + fi } export -f run_idl_check_compatibility find idls -maxdepth 1 -mindepth 1 -type d | xargs -n 1 -P 0 -I % bash -c 'run_idl_check_compatibility "$@"' _ % diff --git a/evergreen/check_legacy_command_types.sh b/evergreen/check_legacy_command_types.sh index 552d0bfe9d7..c5c0d699a7b 100644 --- a/evergreen/check_legacy_command_types.sh +++ b/evergreen/check_legacy_command_types.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src diff --git a/evergreen/check_resmoke_failure.sh b/evergreen/check_resmoke_failure.sh index 4718098b70c..aefa23bcc7b 100755 --- a/evergreen/check_resmoke_failure.sh +++ b/evergreen/check_resmoke_failure.sh @@ -1,5 +1,5 @@ set -o verbose cd src if [ -f resmoke_error_code ]; then - exit $(cat resmoke_error_code) + exit $(cat resmoke_error_code) fi diff --git a/evergreen/check_run_tests_infrastructure_failure.sh b/evergreen/check_run_tests_infrastructure_failure.sh index ecb0ea0f04b..a7cd45c121d 100755 --- a/evergreen/check_run_tests_infrastructure_failure.sh +++ b/evergreen/check_run_tests_infrastructure_failure.sh @@ -1,5 +1,5 @@ set -o verbose cd src if [ -f run_tests_infrastructure_failure ]; then - exit $(cat run_tests_infrastructure_failure) + exit $(cat run_tests_infrastructure_failure) fi diff --git a/evergreen/check_workstation_setup.sh b/evergreen/check_workstation_setup.sh index 71dba730c9a..5ed2a5c124b 100755 --- a/evergreen/check_workstation_setup.sh +++ b/evergreen/check_workstation_setup.sh @@ -2,57 +2,57 @@ failed_setup=false source ~/.bashrc -if command -v pipx &> /dev/null; then - echo "'pipx' command exists" +if command -v pipx &>/dev/null; then + echo "'pipx' command exists" else - echo "pipx command not found - failed setup" - failed_setup=true + echo "pipx command not found - failed setup" + failed_setup=true fi -if command -v poetry &> /dev/null; then - echo "'poetry' command exists" +if command -v poetry &>/dev/null; then + echo "'poetry' command exists" else - echo "poetry command not found - failed setup" - failed_setup=true + echo "poetry command not found - failed setup" + failed_setup=true fi -if command -v db-contrib-tool &> /dev/null; then - echo "'db-contrib-tool' command exists" +if command -v db-contrib-tool &>/dev/null; then + echo "'db-contrib-tool' command exists" else - echo "db-contrib-tool command not found - failed setup" - failed_setup=true + echo "db-contrib-tool command not found - failed setup" + failed_setup=true fi if test -d "./python3-venv"; then - echo "Venv directory exists, checking activation" - . python3-venv/bin/activate - ./buildscripts/resmoke.py run --help &> /dev/null - if [ $? -eq 0 ]; then - echo "Virtual workstation set up correctly" - else - echo "Virtual workstation failed activation" - failed_setup=true - fi - deactivate + echo "Venv directory exists, checking activation" + . python3-venv/bin/activate + ./buildscripts/resmoke.py run --help &>/dev/null + if [ $? -eq 0 ]; then + echo "Virtual workstation set up correctly" + else + echo "Virtual workstation failed activation" + failed_setup=true + fi + deactivate else - echo "mongo virtual environment not created correctly - failed setup" - failed_setup=true + echo "mongo virtual environment not created correctly - failed setup" + failed_setup=true fi if test -d "../Boost-Pretty-Printer"; then - echo "Pretty printers set up correctly" + echo "Pretty printers set up correctly" else - echo "Pretty printers failed setup" - failed_setup=true + echo "Pretty printers failed setup" + failed_setup=true fi if test -f "./compile_commands.json"; then - echo "Clang configuration set up correctly" + echo "Clang configuration set up correctly" else - echo "Clang configuration failed setup" - failed_setup=true + echo "Clang configuration failed setup" + failed_setup=true fi if $failed_setup; then - exit 1 + exit 1 fi diff --git a/evergreen/cleanup_environment.sh b/evergreen/cleanup_environment.sh index 4efc9f8d346..de66f955f9f 100755 --- a/evergreen/cleanup_environment.sh +++ b/evergreen/cleanup_environment.sh @@ -1,13 +1,13 @@ set -o verbose rm -rf \ - /data/db/* \ - mongo-diskstats* \ - mongo-*.tgz \ - ~/.aws \ - ~/.boto \ - venv \ - /data/install \ - /data/multiversion + /data/db/* \ + mongo-diskstats* \ + mongo-*.tgz \ + ~/.aws \ + ~/.boto \ + venv \ + /data/install \ + /data/multiversion exit 0 diff --git a/evergreen/compile_ninja.sh b/evergreen/compile_ninja.sh index a50ec6c3de4..a216e08ad8b 100644 --- a/evergreen/compile_ninja.sh +++ b/evergreen/compile_ninja.sh @@ -1,12 +1,12 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src activate_venv if [ "Windows_NT" = "$OS" ]; then - vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/" - cd "$vcvars" && cmd /K "vcvarsall.bat amd64 && cd ${workdir}\src" + vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/" + cd "$vcvars" && cmd /K "vcvarsall.bat amd64 && cd ${workdir}\src" fi python -m pip install ninja ninja install-core diff --git a/evergreen/compiled_binaries_get.sh b/evergreen/compiled_binaries_get.sh index d3c96c3a385..543dd62177d 100755 --- a/evergreen/compiled_binaries_get.sh +++ b/evergreen/compiled_binaries_get.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -20,19 +20,19 @@ version=${project#mongodb-mongo-} version=${version#v} if [ ! -z "${multiversion_platform_50_or_later}" ]; then - platform="${multiversion_platform_50_or_later}" + platform="${multiversion_platform_50_or_later}" fi # This is primarily for tests for infrastructure which don't always need the latest # binaries. db-contrib-tool setup-repro-env \ - --installDir /data/install \ - --linkDir /data/multiversion \ - --edition $edition \ - --platform $platform \ - --architecture $architecture \ - --evgVersionsFile multiversion-downloads.json \ - $version + --installDir /data/install \ + --linkDir /data/multiversion \ + --edition $edition \ + --platform $platform \ + --architecture $architecture \ + --evgVersionsFile multiversion-downloads.json \ + $version dist_test_dir=$(find /data/install -type d -iname "dist-test") mv "$dist_test_dir" "$(pwd)" diff --git a/evergreen/consolidate_benchmark_txt.sh b/evergreen/consolidate_benchmark_txt.sh index 4d3cbdc08dd..eca05db62c0 100755 --- a/evergreen/consolidate_benchmark_txt.sh +++ b/evergreen/consolidate_benchmark_txt.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -10,5 +10,5 @@ OUTPUT_FILE="build/benchmarks.txt" # Concatenate all text files in the directory into the output file for file in build/*_bm.txt; do - cat "$file" >> "$OUTPUT_FILE" + cat "$file" >>"$OUTPUT_FILE" done diff --git a/evergreen/container_registry_login.sh b/evergreen/container_registry_login.sh index 5b3a32d9348..232031d0415 100644 --- a/evergreen/container_registry_login.sh +++ b/evergreen/container_registry_login.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -8,7 +8,7 @@ attempts=0 max_attempts=4 while ! aws ecr get-login-password --region us-east-1 | podman login --password-stdin --username ${release_tools_container_registry_username_ecr} ${release_tools_container_registry_ecr}; do - [ "$attempts" -ge "$max_attempts" ] && exit 1 - ((attempts++)) - sleep 10 + [ "$attempts" -ge "$max_attempts" ] && exit 1 + ((attempts++)) + sleep 10 done diff --git a/evergreen/coverity_build.sh b/evergreen/coverity_build.sh index 2198bdc9843..87c5c17cad2 100644 --- a/evergreen/coverity_build.sh +++ b/evergreen/coverity_build.sh @@ -10,7 +10,7 @@ python buildscripts/install_bazel.py bazel_bin="$HOME/.local/bin/bazelisk" # number of parallel jobs to use for build. # Even with scale=0 (the default), bc command adds decimal digits in case of multiplication. Division by 1 gives us a whole number with scale=0 -bazel_jobs=$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) * .85 / 1") +bazel_jobs=$(bc <<<"$(grep -c '^processor' /proc/cpuinfo) * .85 / 1") build_config="--config=local --jobs=$bazel_jobs --compiler_type=gcc --opt=off --dbg=False --allocator=system" bazel_query='mnemonic("CppCompile|LinkCompile", filter(//src/mongo, deps(//:install-core)) except //src/mongo/db/modules/enterprise/src/streams/third_party/...)' bazel_cache="--output_user_root=$workdir/bazel_cache" @@ -21,14 +21,14 @@ bazelBuildCommand="$bazel_bin $bazel_cache build $build_config //src/mongo/db/mo echo "Bazel Build Command: $bazelBuildCommand" covIdir="$workdir/covIdir" if [ -d "$covIdir" ]; then - echo "covIdir already exists, meaning idir extracted after download from S3" + echo "covIdir already exists, meaning idir extracted after download from S3" else - mkdir $workdir/covIdir + mkdir $workdir/covIdir fi $workdir/coverity/bin/cov-build --dir "$covIdir" --verbose 0 -j $bazel_jobs --return-emit-failures --parse-error-threshold=99 --bazel $bazelBuildCommand ret=$? if [ $ret -ne 0 ]; then - echo "cov-build faild with exit code $ret" + echo "cov-build faild with exit code $ret" else - echo "cov-build was successful" + echo "cov-build was successful" fi diff --git a/evergreen/crypt_run_tests.sh b/evergreen/crypt_run_tests.sh index 5f0f17aca98..e3c68b05608 100755 --- a/evergreen/crypt_run_tests.sh +++ b/evergreen/crypt_run_tests.sh @@ -3,7 +3,7 @@ # This script verifies that specific symbols, and specific symbols only are # exported in mongo_crypt_v1.so -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -12,8 +12,8 @@ set -o errexit set -o verbose if [ "$(uname)" != "Linux" ]; then - echo "Skipping test, this is for linux only" - exit 0 + echo "Skipping test, this is for linux only" + exit 0 fi EXTRACT_DIR="bazel-bin/install" @@ -25,8 +25,8 @@ GDB_PATH="/opt/mongodbtoolchain/v5/bin/gdb" find $EXTRACT_DIR if [ ! -f "$SOPATH" ]; then - echo "Error: can not find library at: $SOPATH" - exit 1 + echo "Error: can not find library at: $SOPATH" + exit 1 fi # @@ -51,10 +51,10 @@ mongo_crypt_v1_status_get_explanation@@MONGO_CRYPT_1.0' actual="$(readelf -W --dyn-syms "$SOPATH" | awk '$5 == "GLOBAL" && $7 != "UND" && $7 != "ABS" {print $8}' | sort)" if [ "$actual" != "$expect" ]; then - echo "Error: symbols are not as expected in: $SOPATH" - echo "Diff:" - diff <(echo "$actual") <(echo "$expect") - exit 1 + echo "Error: symbols are not as expected in: $SOPATH" + echo "Diff:" + diff <(echo "$actual") <(echo "$expect") + exit 1 fi echo "Mongo Crypt Shared Library exported symbols test succeeded!" @@ -64,8 +64,8 @@ echo "Mongo Crypt Shared Library exported symbols test succeeded!" # and the verify it can be debugged with gdb # if [ ! -f "$UNITTEST_PATH" ]; then - echo "Skipping Mongo Crypt Shared Library unit test. Test not found at $UNITTEST_PATH" - exit 0 + echo "Skipping Mongo Crypt Shared Library unit test. Test not found at $UNITTEST_PATH" + exit 0 fi echo "Running Mongo Crypt Shared Library unit test" @@ -73,8 +73,8 @@ $UNITTEST_PATH echo "Mongo Crypt Shared Library unit test succeeded!" if [ ! -f "$GDB_PATH" ]; then - echo "Skipping Mongo Crypt Shared Library debuggability test. No gdb found at $GDB_PATH" - exit 0 + echo "Skipping Mongo Crypt Shared Library debuggability test. No gdb found at $GDB_PATH" + exit 0 fi echo "Running Mongo Crypt Shared Library debuggability test" diff --git a/evergreen/do_jepsen_setup/create_fuzz_config.sh b/evergreen/do_jepsen_setup/create_fuzz_config.sh index 9fcc414abb8..9c0ec5f6a09 100644 --- a/evergreen/do_jepsen_setup/create_fuzz_config.sh +++ b/evergreen/do_jepsen_setup/create_fuzz_config.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" cd src diff --git a/evergreen/do_jepsen_setup/nodes.sh b/evergreen/do_jepsen_setup/nodes.sh index 650f4fb1dfa..0df79badc1e 100644 --- a/evergreen/do_jepsen_setup/nodes.sh +++ b/evergreen/do_jepsen_setup/nodes.sh @@ -1,8 +1,8 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" cd src set -o errexit activate_venv -$python -c 'import socket; num_nodes = 5; print("\n".join(["%s:%d" % (socket.gethostname(), port) for port in range(20000, 20000 + num_nodes)]))' > nodes.txt +$python -c 'import socket; num_nodes = 5; print("\n".join(["%s:%d" % (socket.gethostname(), port) for port in range(20000, 20000 + num_nodes)]))' >nodes.txt diff --git a/evergreen/engflow_workspace_status.sh b/evergreen/engflow_workspace_status.sh index 99499242aa4..1ca4702470d 100755 --- a/evergreen/engflow_workspace_status.sh +++ b/evergreen/engflow_workspace_status.sh @@ -6,9 +6,9 @@ echo BUILD_SCM_REVISION $(git rev-parse --verify HEAD) git diff-index --quiet HEAD -- if [[ $? == 0 ]]; then - status="clean" + status="clean" else - status="modified" + status="modified" fi echo BUILD_SCM_STATUS $status echo BUILD_SCM_REMOTE git@github.com:10gen/mongo.git diff --git a/evergreen/external_auth_aws_setup.sh b/evergreen/external_auth_aws_setup.sh index 233da94c338..5c3219854f8 100755 --- a/evergreen/external_auth_aws_setup.sh +++ b/evergreen/external_auth_aws_setup.sh @@ -1,10 +1,10 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src set -o errexit -cat << EOF > aws_e2e_setup.json +cat <aws_e2e_setup.json { "iam_auth_ecs_account" : "${iam_auth_ecs_account}", "iam_auth_ecs_secret_access_key" : "${iam_auth_ecs_secret_access_key}", diff --git a/evergreen/external_auth_azure_setup.sh b/evergreen/external_auth_azure_setup.sh index 76909fbe7d5..4e9bb578632 100644 --- a/evergreen/external_auth_azure_setup.sh +++ b/evergreen/external_auth_azure_setup.sh @@ -1,10 +1,10 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src set -o errexit -cat << EOF > $HOME/azure_e2e_config.json +cat <$HOME/azure_e2e_config.json { "tD548GwE1@outlook.com" : "${oidc_azure_test_user_account_one_secret}", "tD548GwE2@outlook.com" : "${oidc_azure_test_user_account_two_secret}", @@ -24,12 +24,12 @@ cat << EOF > $HOME/azure_e2e_config.json "oidc_azure_managed_identity_api_version": "${oidc_azure_managed_identity_api_version}" } EOF -cat << EOF > $HOME/oidc_azure_container_key +cat <$HOME/oidc_azure_container_key ${oidc_azure_container_key} EOF # EVG project variables do not preserve line breaks so we store them as base64 and decode here -sed s/[[:space:]]//g $HOME/oidc_azure_container_key | base64 --decode > $HOME/azure_remote_key +sed s/[[:space:]]//g $HOME/oidc_azure_container_key | base64 --decode >$HOME/azure_remote_key # Clean up temp file rm -f $HOME/oidc_azure_container_key diff --git a/evergreen/external_auth_azure_teardown.sh b/evergreen/external_auth_azure_teardown.sh index 03b8e7ce0d6..4cb75ec08d5 100644 --- a/evergreen/external_auth_azure_teardown.sh +++ b/evergreen/external_auth_azure_teardown.sh @@ -1,11 +1,11 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" set -o errexit # Only run this script for the external_auth_oidc_azure task. if [ "${task_name}" != "external_auth_oidc_azure" ]; then - exit 0 + exit 0 fi echo "Cleaning up Azure OIDC test artifacts" @@ -14,20 +14,20 @@ cd src # Clean up the SSH keyfile, if it exists if [ -f "${HOME}/oidc_azure_container_key" ]; then - rm -f $HOME/oidc_azure_container_key - echo "Cleaned up container key" + rm -f $HOME/oidc_azure_container_key + echo "Cleaned up container key" fi python src/mongo/db/modules/enterprise/jstests/external_auth_oidc_azure/lib/toggle_ingress.py disable --config_file=$HOME/azure_e2e_config.json --lock_file=/tmp/azure_oidc.lock # Clean up the config file, if it exists if [ -f "${HOME}/azure_e2e_config.json" ]; then - rm -f $HOME/azure_e2e_config.json - echo "Cleaned up azure_e2e_config.json" + rm -f $HOME/azure_e2e_config.json + echo "Cleaned up azure_e2e_config.json" fi # Clean up the lock file, if it exists if [ -f "/tmp/azure_oidc.lock" ]; then - rm -f /tmp/azure_oidc.lock - echo "Cleaned up /tmp/azure_oidc.lock" + rm -f /tmp/azure_oidc.lock + echo "Cleaned up /tmp/azure_oidc.lock" fi diff --git a/evergreen/external_auth_gcp_setup.sh b/evergreen/external_auth_gcp_setup.sh index 3dc4a8b3624..246f648b2c4 100755 --- a/evergreen/external_auth_gcp_setup.sh +++ b/evergreen/external_auth_gcp_setup.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -7,7 +7,7 @@ set -o errexit # Create the config file, which will contain the GCE project/zone information along with # the expected audience that will appear on the VM's ID token. -cat << EOF > $HOME/gce_vm_config.json +cat <$HOME/gce_vm_config.json { "audience" : "${oidc_gcp_vm_id_token_audience}", "projectID" : "${oidc_gcp_project_id}", @@ -19,7 +19,7 @@ EOF # Create the SSH key file. Note that the SSH key has been base64 encoded and stored into an EVG # environment variable, so it is first trimmed of any whitespace via sed and base64 decoded before # being output to the file. -echo ${oidc_gcp_ssh_key} | sed "s/[[:space:]]//g" | base64 --decode > $HOME/gcp_ssh_key +echo ${oidc_gcp_ssh_key} | sed "s/[[:space:]]//g" | base64 --decode >$HOME/gcp_ssh_key # Reduce SSH keyfile privileges so that it is secure enough for OpenSSH. chmod 600 $HOME/gcp_ssh_key @@ -34,7 +34,7 @@ ls -al $HOME/gcp_ssh_key # The contents of this file are expected to exist in base64 encoded format in # $oidc_gcp_service_account_key, so the same steps are taken as above before dumping it into a # newly-created JSON file. -echo ${oidc_gcp_service_account_key} | sed "s/[[:space:]]//g" | base64 --decode > ${GOOGLE_APPLICATION_CREDENTIALS} +echo ${oidc_gcp_service_account_key} | sed "s/[[:space:]]//g" | base64 --decode >${GOOGLE_APPLICATION_CREDENTIALS} chmod 600 ${GOOGLE_APPLICATION_CREDENTIALS} ls -al ${GOOGLE_APPLICATION_CREDENTIALS} diff --git a/evergreen/external_auth_gcp_teardown.sh b/evergreen/external_auth_gcp_teardown.sh index f6e72fcf77f..327a1b6d2b7 100755 --- a/evergreen/external_auth_gcp_teardown.sh +++ b/evergreen/external_auth_gcp_teardown.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -7,27 +7,27 @@ set -o errexit # Only run this script for the external_auth_oidc_gcp task. if [ "${task_name}" != "external_auth_oidc_gcp" ]; then - exit 0 + exit 0 fi echo "Cleaning up OIDC GCP test artifacts" # Delete the GCP VM specified in gce_vm_info.json if GOOGLE_APPLICATION_CREDENTIALS is set, points # to a file, and the GCE config and VM info files exist. -if [ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" ] \ - && [ -f "${GOOGLE_APPLICATION_CREDENTIALS}" ] \ - && [ -f "${HOME}/gce_vm_config.json" ] \ - && [ -f "${HOME}/gce_vm_info.json" ]; then - # Install google-cloud-compute so that the script can run. - $python -m pip install google-cloud-compute - $python src/mongo/db/modules/enterprise/jstests/external_auth_oidc_gcp/lib/gce_vm_manager.py delete --config_file $HOME/gce_vm_config.json --service_account_key_file ${GOOGLE_APPLICATION_CREDENTIALS} --output_file $HOME/gce_vm_info.json +if [ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" ] && + [ -f "${GOOGLE_APPLICATION_CREDENTIALS}" ] && + [ -f "${HOME}/gce_vm_config.json" ] && + [ -f "${HOME}/gce_vm_info.json" ]; then + # Install google-cloud-compute so that the script can run. + $python -m pip install google-cloud-compute + $python src/mongo/db/modules/enterprise/jstests/external_auth_oidc_gcp/lib/gce_vm_manager.py delete --config_file $HOME/gce_vm_config.json --service_account_key_file ${GOOGLE_APPLICATION_CREDENTIALS} --output_file $HOME/gce_vm_info.json fi # Clean up the SSH and service account keys if they exist. if [ -f "${HOME}/gcp_ssh_key" ]; then - rm -f $HOME/gcp_ssh_key + rm -f $HOME/gcp_ssh_key fi if [ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" ] && [ -f "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then - rm -f ${GOOGLE_APPLICATION_CREDENTIALS} + rm -f ${GOOGLE_APPLICATION_CREDENTIALS} fi diff --git a/evergreen/external_auth_oidc_setup.sh b/evergreen/external_auth_oidc_setup.sh index 69771c34253..8b20cd31a7c 100755 --- a/evergreen/external_auth_oidc_setup.sh +++ b/evergreen/external_auth_oidc_setup.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -6,7 +6,7 @@ cd src set -o errexit # Should output contents to new file in home directory. -cat << EOF > $HOME/oidc_e2e_setup.json +cat <$HOME/oidc_e2e_setup.json { "testserversecurityone@ping-test.com" : "${oidc_ping_test_user_account_one_secret}", "testserversecuritytwo@ping-test.com" : "${oidc_ping_test_user_account_two_secret}", diff --git a/evergreen/external_auth_oidc_teardown.sh b/evergreen/external_auth_oidc_teardown.sh index 5216e93c6c9..8389de278fe 100644 --- a/evergreen/external_auth_oidc_teardown.sh +++ b/evergreen/external_auth_oidc_teardown.sh @@ -1,15 +1,15 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" # Only run this script for the external_auth_oidc_azure task. if [ "${task_name}" != "external_auth_oidc" ]; then - exit 0 + exit 0 fi echo "Cleaning up OIDC Okta test artifacts" #Clean up the config file, if it exists if [ -f "${HOME}/oidc_e2e_setup.json" ]; then - rm -f $HOME/oidc_e2e_setup.json - echo "Cleaned up oidc_e2e_setup.json" + rm -f $HOME/oidc_e2e_setup.json + echo "Cleaned up oidc_e2e_setup.json" fi diff --git a/evergreen/failed_unittests_gather.sh b/evergreen/failed_unittests_gather.sh index e31e9512008..2551d5ca0fb 100644 --- a/evergreen/failed_unittests_gather.sh +++ b/evergreen/failed_unittests_gather.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -6,88 +6,88 @@ cd src set -eou pipefail # Only run on unit test tasks so we don't target mongod binaries from cores. -if [ "${task_name}" != "run_dbtest" ] \ - && [[ ${task_name} != integration_tests* ]] \ - && [[ "${task_name}" != unit_test_group*_no_sandbox ]]; then - echo "Not gathering failed unittests binaries as this is not a unittest task: ${task_name}" - exit 0 +if [ "${task_name}" != "run_dbtest" ] && + [[ ${task_name} != integration_tests* ]] && + [[ "${task_name}" != unit_test_group*_no_sandbox ]]; then + echo "Not gathering failed unittests binaries as this is not a unittest task: ${task_name}" + exit 0 fi unittest_bin_dir=dist-unittests/bin mkdir -p $unittest_bin_dir || true # Find all core files -core_files=$(/usr/bin/find -H . \( -name "dump_*.core" -o -name "*.mdmp" \) 2> /dev/null) +core_files=$(/usr/bin/find -H . \( -name "dump_*.core" -o -name "*.mdmp" \) 2>/dev/null) while read -r core_file; do - # A core file name does not always have the executable name that generated it. - # See http://stackoverflow.com/questions/34801353/core-dump-filename-gets-thread-name-instead-of-executable-name-with-core-pattern - # On platforms with GDB, we get the binary name from core file - gdb=/opt/mongodbtoolchain/v5/bin/gdb - if [ -f $gdb ]; then - binary_file=$($gdb -batch --quiet -ex "core $core_file" 2> /dev/null | grep "Core was generated" | cut -f2 -d "\`" | cut -f1 -d "'" | cut -f1 -d " ") - binary_file_locations=$binary_file - else - echo "Checking core file '$core_file'" - # Find the base file name from the core file name, note it may be truncated. - # Remove leading 'dump_' and trailing '..core' or '..mdmp' - binary_file=$(echo "$core_file" | sed "s/.*\///;s/dump_//;s/\..*\.core//;s/\..*\.mdmp//") - # Locate the binary file. Since the base file name might be truncated, the find - # may return more than 1 file. - if [ "$binary_file" != "" ]; then - binary_file_locations=$(/usr/bin/find -H . -executable -name "$binary_file*${exe}" 2> /dev/null) - fi - fi - - if [ -z "$binary_file_locations" ]; then - echo "Cannot locate the unittest binary file ($binary_file) that generated the core file $core_file" - else - echo "Files to save: $binary_file_locations" - fi - - for binary_file_location in $binary_file_locations; do - new_binary_file=$unittest_bin_dir/$(echo "$binary_file_location" | sed "s/.*\///") - if [ -f "$binary_file_location" ] && [ ! -f "$new_binary_file" ]; then - echo "Direct Copy $binary_file_location to $new_binary_file" - cp "$binary_file_location" "$new_binary_file" + # A core file name does not always have the executable name that generated it. + # See http://stackoverflow.com/questions/34801353/core-dump-filename-gets-thread-name-instead-of-executable-name-with-core-pattern + # On platforms with GDB, we get the binary name from core file + gdb=/opt/mongodbtoolchain/v5/bin/gdb + if [ -f $gdb ]; then + binary_file=$($gdb -batch --quiet -ex "core $core_file" 2>/dev/null | grep "Core was generated" | cut -f2 -d "\`" | cut -f1 -d "'" | cut -f1 -d " ") + binary_file_locations=$binary_file + else + echo "Checking core file '$core_file'" + # Find the base file name from the core file name, note it may be truncated. + # Remove leading 'dump_' and trailing '..core' or '..mdmp' + binary_file=$(echo "$core_file" | sed "s/.*\///;s/dump_//;s/\..*\.core//;s/\..*\.mdmp//") + # Locate the binary file. Since the base file name might be truncated, the find + # may return more than 1 file. + if [ "$binary_file" != "" ]; then + binary_file_locations=$(/usr/bin/find -H . -executable -name "$binary_file*${exe}" 2>/dev/null) + fi fi - # On Windows if a .pdb symbol file exists, include it in the archive. - if [[ "$binary_file_location" == *".exe" ]]; then - pdb_file=$(echo "$binary_file_location" | sed "s/\.exe/.pdb/") - if [ -f "$pdb_file" ]; then - new_pdb_file=$unittest_bin_dir/$(echo "$pdb_file" | sed "s/.*\///") - echo "PDB Copy $pdb_file to $new_pdb_file" - cp "$pdb_file" "$new_pdb_file" - fi + if [ -z "$binary_file_locations" ]; then + echo "Cannot locate the unittest binary file ($binary_file) that generated the core file $core_file" + else + echo "Files to save: $binary_file_locations" fi - # On binutils platforms, if a .debug symbol file exists, include it - # in the archive - debug_file=$binary_file_location.debug - if [ -f "$debug_file" ]; then - echo "debug Copy $debug_file to $unittest_bin_dir" - cp "$debug_file" "$unittest_bin_dir" - fi + for binary_file_location in $binary_file_locations; do + new_binary_file=$unittest_bin_dir/$(echo "$binary_file_location" | sed "s/.*\///") + if [ -f "$binary_file_location" ] && [ ! -f "$new_binary_file" ]; then + echo "Direct Copy $binary_file_location to $new_binary_file" + cp "$binary_file_location" "$new_binary_file" + fi - # Include any dwp symbol files to go with the .debug files - dwp_file=$binary_file_location.dwp - if [ -f "$dwp_file" ]; then - echo "dwp Copy $dwp_file to $unittest_bin_dir" - cp "$dwp_file" "$unittest_bin_dir" - fi + # On Windows if a .pdb symbol file exists, include it in the archive. + if [[ "$binary_file_location" == *".exe" ]]; then + pdb_file=$(echo "$binary_file_location" | sed "s/\.exe/.pdb/") + if [ -f "$pdb_file" ]; then + new_pdb_file=$unittest_bin_dir/$(echo "$pdb_file" | sed "s/.*\///") + echo "PDB Copy $pdb_file to $new_pdb_file" + cp "$pdb_file" "$new_pdb_file" + fi + fi - # On macOS, these are called .dSYM and they are directories - dsym_dir=$binary_file_location.dSYM - if [ -d "$dsym_dir" ]; then - echo "dsym Copy $dsym_dir to $unittest_bin_dir" - cp -r "$dsym_dir" "$unittest_bin_dir" - fi + # On binutils platforms, if a .debug symbol file exists, include it + # in the archive + debug_file=$binary_file_location.debug + if [ -f "$debug_file" ]; then + echo "debug Copy $debug_file to $unittest_bin_dir" + cp "$debug_file" "$unittest_bin_dir" + fi - done -done <<< "${core_files}" + # Include any dwp symbol files to go with the .debug files + dwp_file=$binary_file_location.dwp + if [ -f "$dwp_file" ]; then + echo "dwp Copy $dwp_file to $unittest_bin_dir" + cp "$dwp_file" "$unittest_bin_dir" + fi + + # On macOS, these are called .dSYM and they are directories + dsym_dir=$binary_file_location.dSYM + if [ -d "$dsym_dir" ]; then + echo "dsym Copy $dsym_dir to $unittest_bin_dir" + cp -r "$dsym_dir" "$unittest_bin_dir" + fi + + done +done <<<"${core_files}" # Copy debug symbols for dynamic builds lib_dir=bazel-bin/install/lib if [ -d "$lib_dir" ] && [ -n "$core_files" ]; then - cp -r "$lib_dir" dist-unittests + cp -r "$lib_dir" dist-unittests fi diff --git a/evergreen/feature_flag_tags_check.sh b/evergreen/feature_flag_tags_check.sh index 8bc96d3eb93..bd9d7ac2a52 100755 --- a/evergreen/feature_flag_tags_check.sh +++ b/evergreen/feature_flag_tags_check.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -18,7 +18,7 @@ mv all_feature_flags.txt patch_all_feature_flags.txt # get the list of feature flags from the base commit git --no-pager diff "$(git merge-base origin/${branch_name} HEAD)" --output="$diff_file_name" --binary if [ -s "$diff_file_name" ]; then - git apply -R "$diff_file_name" + git apply -R "$diff_file_name" fi $python buildscripts/idl/gen_all_feature_flag_list.py turned-on-by-default diff --git a/evergreen/full_disk_debug.sh b/evergreen/full_disk_debug.sh index e21bfb65b38..d287906cf55 100755 --- a/evergreen/full_disk_debug.sh +++ b/evergreen/full_disk_debug.sh @@ -8,17 +8,17 @@ HAS_FULL_DISK=false # 34% /dev/nvme1n1 FILESYSTEMS=$(df -H | grep -vE '^Filesystem|tmpfs|cdrom' | awk '{ print $5 " " $1 }') while read -r output; do - usep=$(echo "$output" | awk '{ print $1}' | cut -d'%' -f1) - partition=$(echo "$output" | awk '{ print $2 }') - if [ $usep -ge $FULL_DISK_THRESHOLD ]; then - echo "Running out of space \"$partition ($usep%)\" on $(hostname) as on $(date)" - HAS_FULL_DISK=true - fi -done <<< "$FILESYSTEMS" + usep=$(echo "$output" | awk '{ print $1}' | cut -d'%' -f1) + partition=$(echo "$output" | awk '{ print $2 }') + if [ $usep -ge $FULL_DISK_THRESHOLD ]; then + echo "Running out of space \"$partition ($usep%)\" on $(hostname) as on $(date)" + HAS_FULL_DISK=true + fi +done <<<"$FILESYSTEMS" if $HAS_FULL_DISK; then - # print all files that are above one megabyte sorted - du -cha / 2> /dev/null | grep -E "^[0-9]+(\.[0-9]+)?[G|M|T]" | sort -h + # print all files that are above one megabyte sorted + du -cha / 2>/dev/null | grep -E "^[0-9]+(\.[0-9]+)?[G|M|T]" | sort -h else - echo "No full partitions found, skipping" + echo "No full partitions found, skipping" fi diff --git a/evergreen/functions/add_git_tag.sh b/evergreen/functions/add_git_tag.sh index c4619317fe5..84ab0b6ed07 100644 --- a/evergreen/functions/add_git_tag.sh +++ b/evergreen/functions/add_git_tag.sh @@ -5,21 +5,21 @@ set -o verbose tag="" if [ -n "$bv_future_git_tag" ]; then - tag="$bv_future_git_tag" + tag="$bv_future_git_tag" fi if [ -n "$future_git_tag" ]; then - tag="$future_git_tag" + tag="$future_git_tag" fi echo "TAG: $tag" if [ -n "$tag" ]; then - if [ "Windows_NT" = "$OS" ]; then - # On Windows, we don't seem to have a local git identity, so we populate the config with this - # dummy email and name. Without a configured email/name, the 'git tag' command will fail. - git config user.email "no-reply@evergreen.@mongodb.com" - git config user.name "Evergreen Agent" - fi + if [ "Windows_NT" = "$OS" ]; then + # On Windows, we don't seem to have a local git identity, so we populate the config with this + # dummy email and name. Without a configured email/name, the 'git tag' command will fail. + git config user.email "no-reply@evergreen.@mongodb.com" + git config user.name "Evergreen Agent" + fi - git tag -a "$tag" -m "$tag" + git tag -a "$tag" -m "$tag" fi diff --git a/evergreen/functions/bolt.sh b/evergreen/functions/bolt.sh index dc1be824065..e19f0dfe4a2 100755 --- a/evergreen/functions/bolt.sh +++ b/evergreen/functions/bolt.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" set +o errexit @@ -6,8 +6,8 @@ set +o errexit cd src if [ -z "${BOLT:-}" ]; then - echo "Not applying BOLT" >&2 - exit 0 + echo "Not applying BOLT" >&2 + exit 0 fi tar -xvf bazel-bin/dist-test-stripped.tgz diff --git a/evergreen/functions/credentials_setup.sh b/evergreen/functions/credentials_setup.sh index 55a9df5282e..fea7d261c6d 100755 --- a/evergreen/functions/credentials_setup.sh +++ b/evergreen/functions/credentials_setup.sh @@ -1,9 +1,9 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" cd src -cat > mci.buildlogger << END_OF_CREDS +cat >mci.buildlogger < /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" set +o errexit @@ -6,8 +6,8 @@ set +o errexit cd src if [ -z "${PGO_PROFILE_URL:-}" ]; then - echo "No pgo profile url specified" >&2 - exit 0 + echo "No pgo profile url specified" >&2 + exit 0 fi wget $PGO_PROFILE_URL diff --git a/evergreen/functions/evergreen_api_credentials_configure.sh b/evergreen/functions/evergreen_api_credentials_configure.sh index 78943d01682..c42f637a2c9 100755 --- a/evergreen/functions/evergreen_api_credentials_configure.sh +++ b/evergreen/functions/evergreen_api_credentials_configure.sh @@ -1,10 +1,10 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" cd src # Create the Evergreen API credentials -cat > .evergreen.yml << END_OF_CREDS +cat >.evergreen.yml < /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" if [ -z "${files}" ]; then - exit 0 + exit 0 fi for file in ${files}; do - if [ -f "$file" ]; then - echo "Removing file $file" - rm -f $file - fi + if [ -f "$file" ]; then + echo "Removing file $file" + rm -f $file + fi done diff --git a/evergreen/functions/fuse_watchdog_cleanup.sh b/evergreen/functions/fuse_watchdog_cleanup.sh index dd6dd587515..2f122a0c018 100755 --- a/evergreen/functions/fuse_watchdog_cleanup.sh +++ b/evergreen/functions/fuse_watchdog_cleanup.sh @@ -1,9 +1,9 @@ cd src if [ -d /data/thrift ]; then - rm -rf /data/thrift + rm -rf /data/thrift fi if [ -d /data/charybdefs ]; then - rm -rf /data/charybdefs + rm -rf /data/charybdefs fi diff --git a/evergreen/functions/get_mongot_version.sh b/evergreen/functions/get_mongot_version.sh index a465ae2ab9c..fa4b5ca90bb 100644 --- a/evergreen/functions/get_mongot_version.sh +++ b/evergreen/functions/get_mongot_version.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" cd src diff --git a/evergreen/functions/modified_patch_files_get_all.sh b/evergreen/functions/modified_patch_files_get_all.sh index 1789d65bdb2..f27bb044662 100755 --- a/evergreen/functions/modified_patch_files_get_all.sh +++ b/evergreen/functions/modified_patch_files_get_all.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" cd src @@ -8,12 +8,12 @@ set -o errexit # For patch builds gather the modified patch files. if [ "${is_patch}" = "true" ]; then - # Get list of patched files - git diff HEAD --name-only >> patch_files.txt - if [ -d src/mongo/db/modules/enterprise ]; then - pushd src/mongo/db/modules/enterprise - # Update the patch_files.txt in the mongo repo. - git diff HEAD --name-only >> ~1/patch_files.txt - popd - fi + # Get list of patched files + git diff HEAD --name-only >>patch_files.txt + if [ -d src/mongo/db/modules/enterprise ]; then + pushd src/mongo/db/modules/enterprise + # Update the patch_files.txt in the mongo repo. + git diff HEAD --name-only >>~1/patch_files.txt + popd + fi fi diff --git a/evergreen/functions/notary_client_credentials_setup.sh b/evergreen/functions/notary_client_credentials_setup.sh index dd03625eb85..c8e963ab109 100755 --- a/evergreen/functions/notary_client_credentials_setup.sh +++ b/evergreen/functions/notary_client_credentials_setup.sh @@ -1,14 +1,14 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" cd src set -o errexit -cat << EOF > notary_env.sh +cat <notary_env.sh export NOTARY_TOKEN=${signing_auth_token_70} export BARQUE_USERNAME=${barque_user} export BARQUE_API_KEY=${barque_api_key} EOF -echo "${signing_auth_token_70}" > signing_auth_token +echo "${signing_auth_token_70}" >signing_auth_token diff --git a/evergreen/functions/process_threads_monitor.sh b/evergreen/functions/process_threads_monitor.sh index 10ae24c343d..083b7822b1c 100755 --- a/evergreen/functions/process_threads_monitor.sh +++ b/evergreen/functions/process_threads_monitor.sh @@ -1,46 +1,46 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" proc_list="(java|lein|mongo|python|_test$|_test\.exe$)" if [ "Windows_NT" = "$OS" ]; then - get_pids() { - proc_pids=$(tasklist /fo:csv \ - | awk -F'","' '{x=$1; gsub("\"","",x); print $2, x}' \ - | grep -iE $1 \ - | cut -f1 -d ' ') - } - get_process_info() { - proc_name="" - proc_info=$(wmic process where "ProcessId=\"$1\"" get "Name,ProcessId,ThreadCount" /format:csv 2> /dev/null | grep $1) - if [ ! -z $proc_info ]; then - proc_name=$(echo $proc_info | cut -f2 -d ',') - proc_threads=$(echo $proc_info | cut -f4 -d ',') - fi - } + get_pids() { + proc_pids=$(tasklist /fo:csv | + awk -F'","' '{x=$1; gsub("\"","",x); print $2, x}' | + grep -iE $1 | + cut -f1 -d ' ') + } + get_process_info() { + proc_name="" + proc_info=$(wmic process where "ProcessId=\"$1\"" get "Name,ProcessId,ThreadCount" /format:csv 2>/dev/null | grep $1) + if [ ! -z $proc_info ]; then + proc_name=$(echo $proc_info | cut -f2 -d ',') + proc_threads=$(echo $proc_info | cut -f4 -d ',') + fi + } else - get_pids() { proc_pids=$(pgrep $1); } - get_process_info() { - proc_name=$(ps -p $1 -o comm=) - # /proc is available on Linux platforms - if [ -f /proc/$1/status ]; then - set_sudo - proc_threads=$($sudo grep Threads /proc/$1/status | sed "s/\s//g" | cut -f2 -d ":") - else - proc_threads=$(ps -AM $1 | grep -vc PID) - fi - } + get_pids() { proc_pids=$(pgrep $1); } + get_process_info() { + proc_name=$(ps -p $1 -o comm=) + # /proc is available on Linux platforms + if [ -f /proc/$1/status ]; then + set_sudo + proc_threads=$($sudo grep Threads /proc/$1/status | sed "s/\s//g" | cut -f2 -d ":") + else + proc_threads=$(ps -AM $1 | grep -vc PID) + fi + } fi while [ 1 ]; do - get_pids $proc_list - if [ ! -z "$proc_pids" ]; then - printf "Running process/thread counter\n" - printf "PROCESS\tPID\tTHREADS\n" - fi - for pid in $proc_pids; do - get_process_info $pid - if [ ! -z "$proc_name" ]; then - printf "$proc_name\t$pid\t$proc_threads\n" + get_pids $proc_list + if [ ! -z "$proc_pids" ]; then + printf "Running process/thread counter\n" + printf "PROCESS\tPID\tTHREADS\n" fi - done - sleep 60 + for pid in $proc_pids; do + get_process_info $pid + if [ ! -z "$proc_name" ]; then + printf "$proc_name\t$pid\t$proc_threads\n" + fi + done + sleep 60 done diff --git a/evergreen/functions/remote_credentials_setup.sh b/evergreen/functions/remote_credentials_setup.sh index 65f6a244816..f93da136341 100644 --- a/evergreen/functions/remote_credentials_setup.sh +++ b/evergreen/functions/remote_credentials_setup.sh @@ -1,16 +1,16 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" # Since the macros 'private_key_remote' and 'private_key_file' are not always defined # we default to /dev/null to avoid syntax errors of an empty expansion. if [ -n "$private_key_remote_bash_var" ]; then - private_key_remote="$private_key_remote_bash_var" + private_key_remote="$private_key_remote_bash_var" fi if [ ! -z "${private_key_remote}" ] && [ ! -z "${private_key_file}" ]; then - mkdir -p ~/.ssh - private_key_file=$(eval echo "$private_key_file") - echo -n "${private_key_remote}" > ${private_key_file} - chmod 0600 ${private_key_file} + mkdir -p ~/.ssh + private_key_file=$(eval echo "$private_key_file") + echo -n "${private_key_remote}" >${private_key_file} + chmod 0600 ${private_key_file} fi # Ensure a clean aws configuration state @@ -24,23 +24,23 @@ aws_profile="${aws_profile_remote}" # The profile in the config file is specified as [profile ], except # for [default], see http://boto3.readthedocs.io/en/latest/guide/configuration.html if [ $aws_profile = "default" ]; then - aws_profile_config="[default]" + aws_profile_config="[default]" else - aws_profile_config="[profile $aws_profile]" + aws_profile_config="[profile $aws_profile]" fi -cat << EOF >> ~/.aws/config +cat <>~/.aws/config $aws_profile_config region = us-east-1 EOF # The profile in the credentials file is specified as []. -cat << EOF >> ~/.aws/credentials +cat <>~/.aws/credentials [$aws_profile] aws_access_key_id = ${aws_key_remote} aws_secret_access_key = ${aws_secret_remote} EOF -cat << EOF > ~/.boto +cat <~/.boto [Boto] https_validate_certificates = False EOF diff --git a/evergreen/functions/resmoke_jobs_determine.sh b/evergreen/functions/resmoke_jobs_determine.sh index 9703078ee32..dd569dc3a66 100755 --- a/evergreen/functions/resmoke_jobs_determine.sh +++ b/evergreen/functions/resmoke_jobs_determine.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" cd src @@ -8,9 +8,9 @@ set -o errexit activate_venv $python buildscripts/evergreen_resmoke_job_count.py \ - --taskName ${task_name} \ - --buildVariant ${build_variant} \ - --distro ${distro_id} \ - --jobFactor ${resmoke_jobs_factor} \ - --jobsMax ${resmoke_jobs_max} \ - --outFile resmoke_jobs_expansion.yml + --taskName ${task_name} \ + --buildVariant ${build_variant} \ + --distro ${distro_id} \ + --jobFactor ${resmoke_jobs_factor} \ + --jobsMax ${resmoke_jobs_max} \ + --outFile resmoke_jobs_expansion.yml diff --git a/evergreen/functions/run_diskstats.sh b/evergreen/functions/run_diskstats.sh index d118f3ae0eb..a507dfa18c9 100755 --- a/evergreen/functions/run_diskstats.sh +++ b/evergreen/functions/run_diskstats.sh @@ -3,20 +3,20 @@ set -o verbose # On Windows we can use typeperf.exe to dump performance counters. if [ "Windows_NT" = "$OS" ]; then - typeperf -qx PhysicalDisk | grep Disk | grep -v _Total > disk_counters.txt - typeperf -cf disk_counters.txt -si 5 -o mongo-diskstats + typeperf -qx PhysicalDisk | grep Disk | grep -v _Total >disk_counters.txt + typeperf -cf disk_counters.txt -si 5 -o mongo-diskstats # Linux: iostat -t option for timestamp. -elif iostat -tdmx > /dev/null 2>&1; then - iostat -tdmx 5 > mongo-diskstats +elif iostat -tdmx >/dev/null 2>&1; then + iostat -tdmx 5 >mongo-diskstats # OSX: Simulate the iostat timestamp. -elif iostat -d > /dev/null 2>&1; then - iostat -d -w 5 | while IFS= read -r line; do printf '%s %s\n' "$(date +'%m/%d/%Y %H:%M:%S')" "$line" >> mongo-diskstats; done +elif iostat -d >/dev/null 2>&1; then + iostat -d -w 5 | while IFS= read -r line; do printf '%s %s\n' "$(date +'%m/%d/%Y %H:%M:%S')" "$line" >>mongo-diskstats; done # Check if vmstat -t is available. -elif vmstat -td > /dev/null 2>&1; then - vmstat -td 5 > mongo-diskstats +elif vmstat -td >/dev/null 2>&1; then + vmstat -td 5 >mongo-diskstats # Check if vmstat -T d is available. -elif vmstat -T d > /dev/null 2>&1; then - vmstat -T d 5 > mongo-diskstats +elif vmstat -T d >/dev/null 2>&1; then + vmstat -T d 5 >mongo-diskstats else - printf "Cannot collect mongo-diskstats on this platform\n" + printf "Cannot collect mongo-diskstats on this platform\n" fi diff --git a/evergreen/functions/system_resource_info_collect.sh b/evergreen/functions/system_resource_info_collect.sh index fd3310822c3..9b9129e6abf 100755 --- a/evergreen/functions/system_resource_info_collect.sh +++ b/evergreen/functions/system_resource_info_collect.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" cd src diff --git a/evergreen/functions/task_timeout_determine.sh b/evergreen/functions/task_timeout_determine.sh index 0dec36cb7a9..8ca98a66605 100644 --- a/evergreen/functions/task_timeout_determine.sh +++ b/evergreen/functions/task_timeout_determine.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" cd src @@ -9,46 +9,46 @@ set -o errexit # Set the suite name to be the task name by default; unless overridden with the `suite`/`suite_config` expansion. suite_name=${task_name} if [[ -n ${suite_config} ]]; then - suite_name=${suite_config} + suite_name=${suite_config} elif [[ -n ${suite} ]]; then - suite_name=${suite} + suite_name=${suite} fi timeout_factor="" if [[ -n "${exec_timeout_factor}" ]]; then - timeout_factor="--exec-timeout-factor ${exec_timeout_factor}" + timeout_factor="--exec-timeout-factor ${exec_timeout_factor}" fi build_variant_for_timeout=${build_variant} if [[ -n "${burn_in_bypass}" ]]; then - # burn_in_tags may generate new build variants, if we are running on one of those build variants - # we should use the build variant it is based on for determining the timeout. This is stored in - # the `burn_in_bypass` expansion. - build_variant_for_timeout=${burn_in_bypass} + # burn_in_tags may generate new build variants, if we are running on one of those build variants + # we should use the build variant it is based on for determining the timeout. This is stored in + # the `burn_in_bypass` expansion. + build_variant_for_timeout=${burn_in_bypass} fi if [[ -n "${alias}" ]]; then - evg_alias=${alias} + evg_alias=${alias} else - evg_alias="evg-alias-absent" + evg_alias="evg-alias-absent" fi resmoke_test_flags="" if [[ -n "${test_flags}" ]]; then - resmoke_test_flags="--test-flags='${test_flags}'" + resmoke_test_flags="--test-flags='${test_flags}'" fi activate_venv PATH=$PATH:$HOME:/ eval $python buildscripts/evergreen_task_timeout.py \ - $timeout_factor \ - $resmoke_test_flags \ - --install-dir "${install_dir}" \ - --task-name ${task_name} \ - --suite-name ${suite_name} \ - --project ${project} \ - --build-variant $build_variant_for_timeout \ - --evg-alias $evg_alias \ - --timeout ${timeout_secs} \ - --exec-timeout ${exec_timeout_secs} \ - --evg-project-config ${evergreen_config_file_path} \ - --out-file task_timeout_expansions.yml + $timeout_factor \ + $resmoke_test_flags \ + --install-dir "${install_dir}" \ + --task-name ${task_name} \ + --suite-name ${suite_name} \ + --project ${project} \ + --build-variant $build_variant_for_timeout \ + --evg-alias $evg_alias \ + --timeout ${timeout_secs} \ + --exec-timeout ${exec_timeout_secs} \ + --evg-project-config ${evergreen_config_file_path} \ + --out-file task_timeout_expansions.yml diff --git a/evergreen/functions/tmp_directory_umount.sh b/evergreen/functions/tmp_directory_umount.sh index e79bff91ec3..fc953a2842b 100644 --- a/evergreen/functions/tmp_directory_umount.sh +++ b/evergreen/functions/tmp_directory_umount.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" set -o errexit diff --git a/evergreen/functions/ulimit_info_collect.sh b/evergreen/functions/ulimit_info_collect.sh index 9d4c8e13c99..cd4736c1387 100755 --- a/evergreen/functions/ulimit_info_collect.sh +++ b/evergreen/functions/ulimit_info_collect.sh @@ -1,11 +1,11 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" if [ "$(uname)" != "Linux" ] && [ "$(uname)" != "Darwin" ]; then - echo "===== Skipping ulimit dump, OS is: $(uname)." + echo "===== Skipping ulimit dump, OS is: $(uname)." else - echo "===== Collecting soft limits:" - ulimit -Sa - echo "===== Collecting hard limits:" - ulimit -Ha + echo "===== Collecting soft limits:" + ulimit -Sa + echo "===== Collecting hard limits:" + ulimit -Ha fi diff --git a/evergreen/functions/venv_adjust.sh b/evergreen/functions/venv_adjust.sh index af5e84c747e..d0802a5a87c 100644 --- a/evergreen/functions/venv_adjust.sh +++ b/evergreen/functions/venv_adjust.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude_python.sh" set -o errexit @@ -11,36 +11,36 @@ popd ARCH=$(uname -m) if [[ "$ARCH" == "arm64" || "$ARCH" == "aarch64" ]]; then - ARCH="arm64" + ARCH="arm64" elif [[ "$ARCH" == "ppc64le" || "$ARCH" == "ppc64" || "$ARCH" == "ppc" || "$ARCH" == "ppcle" ]]; then - ARCH="ppc64le" + ARCH="ppc64le" elif [[ "$ARCH" == "s390x" || "$ARCH" == "s390" ]]; then - ARCH="s390x" + ARCH="s390x" else - ARCH="x86_64" + ARCH="x86_64" fi # TODO SERVER-105520 # try using downloaded venv once more reliability has been built into venv upload/download if [[ "$ARCH" == "ppc64le" ]]; then - rm -rf $venv_dir - source "$DIR/venv_setup.sh" + rm -rf $venv_dir + source "$DIR/venv_setup.sh" else - # Update virtual env directory in activate script - if [ "Windows_NT" = "$OS" ]; then - sed -i -e "s:VIRTUAL_ENV=\".*\":VIRTUAL_ENV=\"$venv_dir\":" "$venv_dir/Scripts/activate" - else - sed -i -e "s:VIRTUAL_ENV=\".*\":VIRTUAL_ENV=\"$venv_dir\":" "$venv_dir/bin/activate" - fi + # Update virtual env directory in activate script + if [ "Windows_NT" = "$OS" ]; then + sed -i -e "s:VIRTUAL_ENV=\".*\":VIRTUAL_ENV=\"$venv_dir\":" "$venv_dir/Scripts/activate" + else + sed -i -e "s:VIRTUAL_ENV=\".*\":VIRTUAL_ENV=\"$venv_dir\":" "$venv_dir/bin/activate" + fi - # Add back python symlinks on linux platforms - if [ "Windows_NT" = "$OS" ]; then - exit 0 - fi + # Add back python symlinks on linux platforms + if [ "Windows_NT" = "$OS" ]; then + exit 0 + fi - cd "$venv_dir/bin" + cd "$venv_dir/bin" - rm python python3 - ln -s "$python_loc" python3 - ln -s python3 python + rm python python3 + ln -s "$python_loc" python3 + ln -s python3 python fi diff --git a/evergreen/functions/venv_setup.sh b/evergreen/functions/venv_setup.sh index 95b379c7fb9..88430d180c2 100644 --- a/evergreen/functions/venv_setup.sh +++ b/evergreen/functions/venv_setup.sh @@ -1,7 +1,7 @@ # exit immediately if virtualenv is not found set -o errexit -evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)/.." +evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.." . "$evergreen_dir/prelude_workdir.sh" . "$evergreen_dir/prelude_python.sh" @@ -10,7 +10,7 @@ echo "python_loc set to $python_loc" venv_dir="${workdir}/venv" if [ -d "$venv_dir" ]; then - exit 0 + exit 0 fi # We create a venv for poetry @@ -19,9 +19,9 @@ fi # See issue SERVER-80781 POETRY_VENV="${workdir}/poetry_venv" if [ "Windows_NT" = "$OS" ]; then - POETRY_VENV_PYTHON="$POETRY_VENV/Scripts/python.exe" + POETRY_VENV_PYTHON="$POETRY_VENV/Scripts/python.exe" else - POETRY_VENV_PYTHON="$POETRY_VENV/bin/python3" + POETRY_VENV_PYTHON="$POETRY_VENV/bin/python3" fi "$python_loc" -m venv "$POETRY_VENV" @@ -36,20 +36,20 @@ export POETRY_CACHE_DIR="$poetry_dir/cache" export PIP_CACHE_DIR="$poetry_dir/pip_cache" pushd src for i in {1..5}; do - $POETRY_VENV_PYTHON -m pip install -r poetry_requirements.txt && RET=0 && break || RET=$? && sleep 1 - echo "Python failed to install poetry, retrying..." + $POETRY_VENV_PYTHON -m pip install -r poetry_requirements.txt && RET=0 && break || RET=$? && sleep 1 + echo "Python failed to install poetry, retrying..." done popd if [ $RET -ne 0 ]; then - echo "Pip install error for poetry" - exit $RET + echo "Pip install error for poetry" + exit $RET fi "$python_loc" -m venv "$venv_dir" # Adding README file for using this venv locally -cat << EOF >> venv_readme.txt +cat <>venv_readme.txt This is an archive of the Python venv generated by this Evergreen build. You can use it locally to avoid needing to manually set up the Python environment. @@ -60,11 +60,11 @@ echo "Updating virtual env directory in activate script" pushd venv; venv_dir=\$(pwd); popd EOF if [ "Windows_NT" = "$OS" ]; then - cat << EOF >> venv_readme.txt + cat <>venv_readme.txt sed -i -e "s:VIRTUAL_ENV=\".*\":VIRTUAL_ENV=\"\$venv_dir\":" "\$venv_dir/Scripts/activate" EOF else - cat << EOF >> venv_readme.txt + cat <>venv_readme.txt sed -i -e "s:VIRTUAL_ENV=\".*\":VIRTUAL_ENV=\"\$venv_dir\":" "\$venv_dir/bin/activate" echo "Adding back python symlinks" @@ -85,7 +85,7 @@ fi # End of README file # cygwin bash does not like. dos2unix it # (See https://bugs.python.org/issue32451) if [ "Windows_NT" = "$OS" ]; then - dos2unix "${workdir}/venv/Scripts/activate" + dos2unix "${workdir}/venv/Scripts/activate" fi export VIRTUAL_ENV_DISABLE_PROMPT=yes @@ -101,13 +101,13 @@ echo "Upgrading pip to 21.0.1" # We have seen weird network errors that can sometimes mess up the pip install # By retrying we would like to only see errors that happen consistently for i in {1..5}; do - python -m pip --disable-pip-version-check install "pip==21.0.1" "wheel==0.37.0" && RET=0 && break || RET=$? && sleep 1 - echo "Python failed to install pip and wheel, retrying..." + python -m pip --disable-pip-version-check install "pip==21.0.1" "wheel==0.37.0" && RET=0 && break || RET=$? && sleep 1 + echo "Python failed to install pip and wheel, retrying..." done if [ $RET -ne 0 ]; then - echo "Pip install error for wheel and pip version" - exit $RET + echo "Pip install error for wheel and pip version" + exit $RET fi cd src @@ -117,20 +117,20 @@ cd src # By retrying we would like to only see errors that happen consistently count=0 for i in {1..5}; do - yes | $POETRY_VENV_PYTHON -m poetry cache clear . --all - rm -rf $poetry_dir/* - $POETRY_VENV_PYTHON -m poetry install --no-root --sync && RET=0 && break || RET=$? && sleep 1 + yes | $POETRY_VENV_PYTHON -m poetry cache clear . --all + rm -rf $poetry_dir/* + $POETRY_VENV_PYTHON -m poetry install --no-root --sync && RET=0 && break || RET=$? && sleep 1 - echo "Python failed install required deps with poetry, retrying..." - sleep $((count * count * 20)) - count=$((count + 1)) + echo "Python failed install required deps with poetry, retrying..." + sleep $((count * count * 20)) + count=$((count + 1)) done if [ $RET -ne 0 ]; then - echo "Poetry install error for full venv" - exit $RET + echo "Poetry install error for full venv" + exit $RET fi cd .. -python -m pip freeze > pip-requirements.txt +python -m pip freeze >pip-requirements.txt diff --git a/evergreen/functions/version_expansions_generate.sh b/evergreen/functions/version_expansions_generate.sh index 75c89b94420..0fa1180b11c 100755 --- a/evergreen/functions/version_expansions_generate.sh +++ b/evergreen/functions/version_expansions_generate.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" cd src @@ -10,13 +10,13 @@ MONGO_VERSION=$(git describe --abbrev=7) # If the project is sys-perf (or related), add the string -sys-perf to the version if [[ "${project}" == sys-perf* ]]; then - MONGO_VERSION="$MONGO_VERSION-sys-perf" + MONGO_VERSION="$MONGO_VERSION-sys-perf" fi # If this is a patch build, we add the patch version id to the version string so we know # this build was a patch, and which evergreen task it came from if [ "${is_patch}" = "true" ]; then - MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" + MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" fi echo "MONGO_VERSION = ${MONGO_VERSION}" diff --git a/evergreen/functions/wiredtiger_develop_use.sh b/evergreen/functions/wiredtiger_develop_use.sh index 97e16e990c7..59e482bf0aa 100755 --- a/evergreen/functions/wiredtiger_develop_use.sh +++ b/evergreen/functions/wiredtiger_develop_use.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" cd src @@ -6,10 +6,10 @@ cd src set -o errexit set -o verbose if [ "${use_wt_develop}" = "true" ]; then - echo "Using the wtdevelop module instead..." - cd src/third_party - for wtdir in dist examples ext lang src test tools; do - rm -rf wiredtiger/$wtdir - mv wtdevelop/$wtdir wiredtiger/ - done + echo "Using the wtdevelop module instead..." + cd src/third_party + for wtdir in dist examples ext lang src test tools; do + rm -rf wiredtiger/$wtdir + mv wtdevelop/$wtdir wiredtiger/ + done fi diff --git a/evergreen/garasign_gpg_crypt_sign.sh b/evergreen/garasign_gpg_crypt_sign.sh index b7c7ebb0b3c..aa19306f223 100644 --- a/evergreen/garasign_gpg_crypt_sign.sh +++ b/evergreen/garasign_gpg_crypt_sign.sh @@ -1,10 +1,10 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src -echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_80}" >> "signing-envfile" -echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_80}" >> "signing-envfile" +echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_80}" >>"signing-envfile" +echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_80}" >>"signing-envfile" set -o errexit set -o verbose @@ -20,14 +20,14 @@ shasum -a 256 $crypt_file_name | tee $crypt_file_name.sha256 md5sum $crypt_file_name | tee $crypt_file_name.md5 # signing crypt linux artifact with gpg -cat << EOF >> gpg_signing_commands.sh +cat <>gpg_signing_commands.sh gpgloader # loading gpg keys. gpg --yes -v --armor -o $crypt_file_name.sig --detach-sign $crypt_file_name EOF podman run \ - --env-file=signing-envfile \ - --rm \ - -v $(pwd):$(pwd) -w $(pwd) \ - ${garasign_gpg_image_ecr} \ - /bin/bash -c "$(cat ./gpg_signing_commands.sh)" + --env-file=signing-envfile \ + --rm \ + -v $(pwd):$(pwd) -w $(pwd) \ + ${garasign_gpg_image_ecr} \ + /bin/bash -c "$(cat ./gpg_signing_commands.sh)" diff --git a/evergreen/garasign_gpg_sign.sh b/evergreen/garasign_gpg_sign.sh index 2093396a81c..781e5903900 100644 --- a/evergreen/garasign_gpg_sign.sh +++ b/evergreen/garasign_gpg_sign.sh @@ -1,17 +1,17 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src -echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_80}" >> "signing-envfile" -echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_80}" >> "signing-envfile" +echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_80}" >>"signing-envfile" +echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_80}" >>"signing-envfile" set -o errexit set -o verbose long_ext=${ext} if [ "$long_ext" == "tgz" ]; then - long_ext="tar.gz" + long_ext="tar.gz" fi mv mongo-binaries.tgz mongodb-${push_name}-${push_arch}-${suffix}.${ext} @@ -22,13 +22,13 @@ mv distsrc.${ext} mongodb-src-${src_suffix}.${long_ext} || true # generating checksums function gen_checksums() { - if [ -e $1 ]; then - shasum -a 1 $1 | tee $1.sha1 - shasum -a 256 $1 | tee $1.sha256 - md5sum $1 | tee $1.md5 - else - echo "$1 does not exist. Skipping checksum generation" - fi + if [ -e $1 ]; then + shasum -a 1 $1 | tee $1.sha1 + shasum -a 256 $1 | tee $1.sha256 + md5sum $1 | tee $1.md5 + else + echo "$1 does not exist. Skipping checksum generation" + fi } gen_checksums mongodb-$push_name-$push_arch-$suffix.$ext @@ -37,7 +37,7 @@ gen_checksums mongodb-src-$src_suffix.$long_ext gen_checksums mongodb-cryptd-$push_name-$push_arch-$suffix.$ext # signing linux artifacts with gpg -cat << 'EOF' > gpg_signing_commands.sh +cat <<'EOF' >gpg_signing_commands.sh gpgloader # loading gpg keys. function sign(){ if [ -e $1 ] @@ -50,7 +50,7 @@ function sign(){ EOF -cat << EOF >> gpg_signing_commands.sh +cat <>gpg_signing_commands.sh sign mongodb-$push_name-$push_arch-$suffix.$ext sign mongodb-$push_name-$push_arch-debugsymbols-$suffix.$ext sign mongodb-src-$src_suffix.$long_ext @@ -58,8 +58,8 @@ sign mongodb-cryptd-$push_name-$push_arch-$suffix.$ext EOF podman run \ - --env-file=signing-envfile \ - --rm \ - -v $(pwd):$(pwd) -w $(pwd) \ - ${garasign_gpg_image_ecr} \ - /bin/bash -c "$(cat ./gpg_signing_commands.sh)" + --env-file=signing-envfile \ + --rm \ + -v $(pwd):$(pwd) -w $(pwd) \ + ${garasign_gpg_image_ecr} \ + /bin/bash -c "$(cat ./gpg_signing_commands.sh)" diff --git a/evergreen/garasign_jsign_sign.sh b/evergreen/garasign_jsign_sign.sh index a07fd0e7fb3..1cb25735bf8 100644 --- a/evergreen/garasign_jsign_sign.sh +++ b/evergreen/garasign_jsign_sign.sh @@ -1,14 +1,14 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" if [ "${push_name}" != "windows" ]; then - exit 0 + exit 0 fi cd src -echo "GRS_CONFIG_USER1_USERNAME=${garasign_jsign_username}" >> "signing-envfile" -echo "GRS_CONFIG_USER1_PASSWORD=${garasign_jsign_password}" >> "signing-envfile" +echo "GRS_CONFIG_USER1_USERNAME=${garasign_jsign_username}" >>"signing-envfile" +echo "GRS_CONFIG_USER1_PASSWORD=${garasign_jsign_password}" >>"signing-envfile" set -o errexit set -o verbose @@ -17,8 +17,8 @@ msi_filename=mongodb-${push_name}-${push_arch}-${suffix}.msi cp bazel-bin/src/mongo/installer/msi/mongodb-win32-x86_64-windows-${version}.msi $msi_filename if [ "${is_patch}" != "true" ]; then - # signing windows artifacts with jsign - cat << 'EOF' > jsign_signing_commands.sh + # signing windows artifacts with jsign + cat <<'EOF' >jsign_signing_commands.sh function sign(){ if [ -e $1 ] then @@ -28,25 +28,25 @@ function sign(){ fi } EOF - cat << EOF >> jsign_signing_commands.sh + cat <>jsign_signing_commands.sh sign $msi_filename EOF - podman run \ - --env-file=signing-envfile \ - --rm \ - -v $(pwd):$(pwd) -w $(pwd) \ - ${garasign_jsign_image_ecr} \ - /bin/bash -c "$(cat ./jsign_signing_commands.sh)" + podman run \ + --env-file=signing-envfile \ + --rm \ + -v $(pwd):$(pwd) -w $(pwd) \ + ${garasign_jsign_image_ecr} \ + /bin/bash -c "$(cat ./jsign_signing_commands.sh)" else - echo "Not signing windows msi due to it being a patch build" + echo "Not signing windows msi due to it being a patch build" fi # generating checksums if [ -e $msi_filename ]; then - shasum -a 1 $msi_filename | tee $msi_filename.sha1 - shasum -a 256 $msi_filename | tee $msi_filename.sha256 - md5sum $msi_filename | tee $msi_filename.md5 + shasum -a 1 $msi_filename | tee $msi_filename.sha1 + shasum -a 256 $msi_filename | tee $msi_filename.sha256 + md5sum $msi_filename | tee $msi_filename.md5 else - echo "$msi_filename does not exist. Skipping checksum generation" + echo "$msi_filename does not exist. Skipping checksum generation" fi diff --git a/evergreen/gather_mongo_coredumps.sh b/evergreen/gather_mongo_coredumps.sh index 7c596aa836c..aa38dc018e6 100755 --- a/evergreen/gather_mongo_coredumps.sh +++ b/evergreen/gather_mongo_coredumps.sh @@ -1,10 +1,10 @@ cd src # Find all core files and move to src -core_files=$(/usr/bin/find -H .. \( -name "*.core" -o -name "*.mdmp" \) 2> /dev/null) +core_files=$(/usr/bin/find -H .. \( -name "*.core" -o -name "*.mdmp" \) 2>/dev/null) for core_file in $core_files; do - base_name=$(echo $core_file | sed "s/.*\///") - # Move file if it does not already exist - if [ ! -f $base_name ]; then - mv $core_file . - fi + base_name=$(echo $core_file | sed "s/.*\///") + # Move file if it does not already exist + if [ ! -f $base_name ]; then + mv $core_file . + fi done diff --git a/evergreen/gen_feature_flags.sh b/evergreen/gen_feature_flags.sh index 5d1bbd722cc..67fa81b5fef 100755 --- a/evergreen/gen_feature_flags.sh +++ b/evergreen/gen_feature_flags.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src diff --git a/evergreen/gen_supplementary_data.sh b/evergreen/gen_supplementary_data.sh index 0d0c003403c..8b7b9c1779d 100755 --- a/evergreen/gen_supplementary_data.sh +++ b/evergreen/gen_supplementary_data.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -8,79 +8,79 @@ set -o verbose activate_venv if [ -z "${build_patch_id}" ] || [ -z "${reuse_compile_from}" ] || [ "${is_patch:-false}" = "false" ]; then - # Create target folder - mkdir -p mongodb/ + # Create target folder + mkdir -p mongodb/ - # Generate feature flag list - $python buildscripts/idl/gen_all_feature_flag_list.py turned-off-by-default - mkdir -p mongodb/feature_flags - cp ./all_feature_flags.txt mongodb/feature_flags + # Generate feature flag list + $python buildscripts/idl/gen_all_feature_flag_list.py turned-off-by-default + mkdir -p mongodb/feature_flags + cp ./all_feature_flags.txt mongodb/feature_flags - # Generate server params list - $python buildscripts/idl/gen_all_server_params_list.py - mkdir -p mongodb/server_params - cp ./all_server_params.txt mongodb/server_params + # Generate server params list + $python buildscripts/idl/gen_all_server_params_list.py + mkdir -p mongodb/server_params + cp ./all_server_params.txt mongodb/server_params - # Download mongo tools - arch=$(uname -m) - if [ -f /etc/os-release ]; then - . /etc/os-release - if [ "$ID" == "amzn" ]; then - case $arch in - "x86_64" | "aarch64") - case $VERSION_ID in - "2" | "2023") - binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon${VERSION_ID}-${arch}-100.9.4.tgz" - ;; - *) - echo "Unsupported Amazon Linux version: $VERSION_ID" - exit 1 - ;; - esac - ;; - *) - echo "Unsupported architecture: $arch" - exit 1 - ;; - esac + # Download mongo tools + arch=$(uname -m) + if [ -f /etc/os-release ]; then + . /etc/os-release + if [ "$ID" == "amzn" ]; then + case $arch in + "x86_64" | "aarch64") + case $VERSION_ID in + "2" | "2023") + binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon${VERSION_ID}-${arch}-100.9.4.tgz" + ;; + *) + echo "Unsupported Amazon Linux version: $VERSION_ID" + exit 1 + ;; + esac + ;; + *) + echo "Unsupported architecture: $arch" + exit 1 + ;; + esac + else + echo "Unsupported Linux distribution: $ID" + exit 1 + fi else - echo "Unsupported Linux distribution: $ID" - exit 1 + echo "Unable to determine Linux distribution" + exit 1 fi - else - echo "Unable to determine Linux distribution" - exit 1 - fi - wget "$binary_url" -O mongo-tools.tar.gz - tar -xzvf mongo-tools.tar.gz -C mongodb/ --strip-components=1 "mong*/bin" + wget "$binary_url" -O mongo-tools.tar.gz + tar -xzvf mongo-tools.tar.gz -C mongodb/ --strip-components=1 "mong*/bin" - # generate atlas info - uarch=$(uname -p) - os=$(uname -r) - json="{ \"version\": \"${version}\", \"gitVersion\": \"${revision}\", \"uarch\": \"$uarch\", \"os\": \"$os\" }" - echo $json | jq '.' > mongodb/atlas_info.json + # generate atlas info + uarch=$(uname -p) + os=$(uname -r) + json="{ \"version\": \"${version}\", \"gitVersion\": \"${revision}\", \"uarch\": \"$uarch\", \"os\": \"$os\" }" + echo $json | jq '.' >mongodb/atlas_info.json - # Add custom run_validate_collections.js wrapper - mv jstests/hooks/run_validate_collections.js jstests/hooks/run_validate_collections.actual.js - cat << EOF > jstests/hooks/run_validate_collections.js + # Add custom run_validate_collections.js wrapper + mv jstests/hooks/run_validate_collections.js jstests/hooks/run_validate_collections.actual.js + cat <jstests/hooks/run_validate_collections.js print("NOTE: run_validate_collections.js will skip the oplog!"); TestData = { skipValidationNamespaces: ['local.oplog.rs'] }; await import("jstests/hooks/run_validate_collections.actual.js"); EOF - # Copy the js tests - mkdir -p mongodb/jstests/hooks - cp -a jstests/* mongodb/jstests + # Copy the js tests + mkdir -p mongodb/jstests/hooks + cp -a jstests/* mongodb/jstests - # Copy the build scripts - mkdir -p mongodb/buildscripts - cp -a buildscripts/* mongodb/buildscripts + # Copy the build scripts + mkdir -p mongodb/buildscripts + cp -a buildscripts/* mongodb/buildscripts - # Create the final archive - tar czf supplementary-data.tgz mongodb + # Create the final archive + tar czf supplementary-data.tgz mongodb else - # Evergreen does not handle nested escaped expansions well - version_to_reuse_from=$(if [ -n "${build_patch_id}" ]; then echo "${build_patch_id}"; else echo "${reuse_compile_from}"; fi) - curl -o supplementary-data.tgz https://s3.amazonaws.com/mciuploads/"${project}"/"${compile_variant}"/"${version_to_reuse_from}"/dsi/supplementary-data.tgz + # Evergreen does not handle nested escaped expansions well + version_to_reuse_from=$(if [ -n "${build_patch_id}" ]; then echo "${build_patch_id}"; else echo "${reuse_compile_from}"; fi) + curl -o supplementary-data.tgz https://s3.amazonaws.com/mciuploads/"${project}"/"${compile_variant}"/"${version_to_reuse_from}"/dsi/supplementary-data.tgz fi diff --git a/evergreen/gen_tasks_activate.sh b/evergreen/gen_tasks_activate.sh index 026f4615fe2..22585842b80 100644 --- a/evergreen/gen_tasks_activate.sh +++ b/evergreen/gen_tasks_activate.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -7,5 +7,5 @@ set -o errexit activate_venv $python buildscripts/evergreen_activate_gen_tasks.py \ - --expansion-file ../expansions.yml \ - --verbose + --expansion-file ../expansions.yml \ + --verbose diff --git a/evergreen/generate_buildid_debug_symbols_mapping.sh b/evergreen/generate_buildid_debug_symbols_mapping.sh index 5260e1f77d8..c582b3792ae 100644 --- a/evergreen/generate_buildid_debug_symbols_mapping.sh +++ b/evergreen/generate_buildid_debug_symbols_mapping.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -8,19 +8,19 @@ set -o verbose build_patch_id="${build_patch_id:-${reuse_compile_from}}" if [ -n "${build_patch_id}" ]; then - exit 0 + exit 0 fi is_san_variant_arg="" if [[ -n "${san_options}" ]]; then - is_san_variant_arg="--is-san-variant" + is_san_variant_arg="--is-san-variant" fi activate_venv $python buildscripts/debugsymb_mapper.py \ - --version "${version_id}" \ - --client-id "${symbolizer_client_id}" \ - --client-secret "${symbolizer_client_secret}" \ - --variant "${build_variant}" \ - $is_san_variant_arg + --version "${version_id}" \ + --client-id "${symbolizer_client_id}" \ + --client-secret "${symbolizer_client_secret}" \ + --variant "${build_variant}" \ + $is_san_variant_arg diff --git a/evergreen/generate_custom_build_promotion_expansions.sh b/evergreen/generate_custom_build_promotion_expansions.sh index c0ea21d4011..b8abc69da89 100755 --- a/evergreen/generate_custom_build_promotion_expansions.sh +++ b/evergreen/generate_custom_build_promotion_expansions.sh @@ -1,10 +1,10 @@ set -o errexit curl --fail-with-body \ - --header "Api-User: ${EVERGREEN_API_USER}" \ - --header "Api-Key: ${EVERGREEN_API_KEY}" \ - -L https://evergreen.mongodb.com/rest/v2/tasks/${PROMOTE_TASK_ID} \ - --output ./task_data.json + --header "Api-User: ${EVERGREEN_API_USER}" \ + --header "Api-Key: ${EVERGREEN_API_KEY}" \ + -L https://evergreen.mongodb.com/rest/v2/tasks/${PROMOTE_TASK_ID} \ + --output ./task_data.json echo ".................." echo "task data" @@ -20,7 +20,7 @@ promote_revision=$(cat task_data.json | jq -r ".revision") artifact_address="https://internal-downloads.mongodb.com/server-custom-builds/${promote_project_id}/${promote_version_id}/${promote_build_variant}/mongo-${promote_build_id}.tgz" -cat << EOT > ./promote-expansions.yml +cat <./promote-expansions.yml promote_project_id: "$promote_project_id" promote_version_id: "$promote_version_id" promote_build_id: "$promote_build_id" diff --git a/evergreen/generate_evergreen_bazelrc.sh b/evergreen/generate_evergreen_bazelrc.sh index 9da388e1683..0238abb1ac7 100644 --- a/evergreen/generate_evergreen_bazelrc.sh +++ b/evergreen/generate_evergreen_bazelrc.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -9,48 +9,48 @@ set -o verbose # Use the Evergreen temp directory to avoid filling up the disk. mkdir -p $TMPDIR if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then - mkdir -p Z:/bazel_tmp - touch Z:/bazel_tmp/mci_path - # TODO(SERVER-94605): remove when Windows temp directory is cleared between task runs - if [[ "$PWD" != "$(cat Z:/bazel_tmp/mci_path)" ]]; then - echo "Clearing bazel output root from previous task mci '$(cat Z:/bazel_tmp/mci_path)'" - rm -rf Z:/bazel_tmp/* || true - echo $PWD > Z:/bazel_tmp/mci_path - fi + mkdir -p Z:/bazel_tmp + touch Z:/bazel_tmp/mci_path + # TODO(SERVER-94605): remove when Windows temp directory is cleared between task runs + if [[ "$PWD" != "$(cat Z:/bazel_tmp/mci_path)" ]]; then + echo "Clearing bazel output root from previous task mci '$(cat Z:/bazel_tmp/mci_path)'" + rm -rf Z:/bazel_tmp/* || true + echo $PWD >Z:/bazel_tmp/mci_path + fi - # Z:/ path is necessary to avoid running into MSVC's file length limit, - # see https://jira.mongodb.org/browse/DEVPROD-11126 - abs_path=$(cygpath -w "$TMPDIR" | tr '\\' '/') - echo "startup --output_user_root=Z:/bazel_tmp" > .bazelrc.evergreen - echo "common --action_env=TMP=Z:/bazel_tmp" >> .bazelrc.evergreen - echo "common --action_env=TEMP=Z:/bazel_tmp" >> .bazelrc.evergreen - echo "BAZELISK_HOME=${abs_path}/bazelisk_home" >> .bazeliskrc - # echo "common --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" >> .bazelrc.git - echo "common --define GIT_COMMIT_HASH=nogitversion" >> .bazelrc.git + # Z:/ path is necessary to avoid running into MSVC's file length limit, + # see https://jira.mongodb.org/browse/DEVPROD-11126 + abs_path=$(cygpath -w "$TMPDIR" | tr '\\' '/') + echo "startup --output_user_root=Z:/bazel_tmp" >.bazelrc.evergreen + echo "common --action_env=TMP=Z:/bazel_tmp" >>.bazelrc.evergreen + echo "common --action_env=TEMP=Z:/bazel_tmp" >>.bazelrc.evergreen + echo "BAZELISK_HOME=${abs_path}/bazelisk_home" >>.bazeliskrc + # echo "common --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" >> .bazelrc.git + echo "common --define GIT_COMMIT_HASH=nogitversion" >>.bazelrc.git else - echo "startup --output_user_root=${TMPDIR}/bazel-output-root" > .bazelrc.evergreen - echo "BAZELISK_HOME=${TMPDIR}/bazelisk_home" >> .bazeliskrc - echo "common --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" >> .bazelrc.git + echo "startup --output_user_root=${TMPDIR}/bazel-output-root" >.bazelrc.evergreen + echo "BAZELISK_HOME=${TMPDIR}/bazelisk_home" >>.bazeliskrc + echo "common --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" >>.bazelrc.git fi if [[ "${evergreen_remote_exec}" != "on" ]]; then - # Temporarily disable remote exec and only use remote cache - echo "common --remote_executor=" >> .bazelrc.evergreen - echo "common --modify_execution_info=.*=+no-remote-exec" >> .bazelrc.evergreen - echo "common --jobs=auto" >> .bazelrc.evergreen + # Temporarily disable remote exec and only use remote cache + echo "common --remote_executor=" >>.bazelrc.evergreen + echo "common --modify_execution_info=.*=+no-remote-exec" >>.bazelrc.evergreen + echo "common --jobs=auto" >>.bazelrc.evergreen fi uri="https://spruce.mongodb.com/task/${task_id:?}?execution=${execution:?}" -echo "common --tls_client_certificate=./engflow.cert" >> .bazelrc.evergreen -echo "common --tls_client_key=./engflow.key" >> .bazelrc.evergreen -echo "common --bes_keywords=engflow:CiCdPipelineName=${build_variant:?}" >> .bazelrc.evergreen -echo "common --bes_keywords=engflow:CiCdJobName=${task_name:?}" >> .bazelrc.evergreen -echo "common --bes_keywords=engflow:CiCdUri=${uri:?}" >> .bazelrc.evergreen -echo "common --bes_keywords=evg:project=${project:?}" >> .bazelrc.evergreen -echo "common --remote_upload_local_results=True" >> .bazelrc.evergreen -echo "common --test_output=summary" >> .bazelrc.evergreen +echo "common --tls_client_certificate=./engflow.cert" >>.bazelrc.evergreen +echo "common --tls_client_key=./engflow.key" >>.bazelrc.evergreen +echo "common --bes_keywords=engflow:CiCdPipelineName=${build_variant:?}" >>.bazelrc.evergreen +echo "common --bes_keywords=engflow:CiCdJobName=${task_name:?}" >>.bazelrc.evergreen +echo "common --bes_keywords=engflow:CiCdUri=${uri:?}" >>.bazelrc.evergreen +echo "common --bes_keywords=evg:project=${project:?}" >>.bazelrc.evergreen +echo "common --remote_upload_local_results=True" >>.bazelrc.evergreen +echo "common --test_output=summary" >>.bazelrc.evergreen # Disable remote execution in evergreen only since it runs on every PR, but we still # want it to be fast on workstations -echo "coverage --config=no-remote-exec" >> .bazelrc.evergreen +echo "coverage --config=no-remote-exec" >>.bazelrc.evergreen diff --git a/evergreen/generate_sast_report.sh b/evergreen/generate_sast_report.sh index 41cffbfe354..cc282ac7d30 100755 --- a/evergreen/generate_sast_report.sh +++ b/evergreen/generate_sast_report.sh @@ -9,19 +9,19 @@ virtualenv -p python3.12 .venv source .venv/bin/activate pip install -r sast_reporting/requirements.txt if [ -z "${TRIGGERED_BY_GIT_TAG}" ]; then - echo "Evergreen version was NOT triggered by a git tag" - echo "Setting Google Drive folder ID for non-release" - google_drive_folder_id="${SAST_REPORT_TEST_GOOGLE_DRIVE_FOLDER_ID}" + echo "Evergreen version was NOT triggered by a git tag" + echo "Setting Google Drive folder ID for non-release" + google_drive_folder_id="${SAST_REPORT_TEST_GOOGLE_DRIVE_FOLDER_ID}" else - echo "Evergreen version was triggered by git tag '${TRIGGERED_BY_GIT_TAG}'" - echo "Setting Google Drive folder ID for release" - google_drive_folder_id="${SAST_REPORT_RELEASES_GOOGLE_DRIVE_FOLDER_ID}" + echo "Evergreen version was triggered by git tag '${TRIGGERED_BY_GIT_TAG}'" + echo "Setting Google Drive folder ID for release" + google_drive_folder_id="${SAST_REPORT_RELEASES_GOOGLE_DRIVE_FOLDER_ID}" fi python3 -m sast_reporting.src.mongodb_server \ - --version ${MONGODB_VERSION} \ - --branch ${MONGODB_RELEASE_BRANCH} \ - --commit-date $commit_datetime \ - --output-path ${MODULE_PATH}/sast_report_${MONGODB_VERSION}.xlsx \ - --upload-file-name "[${MONGODB_VERSION}] MongoDB Server Enterprise SAST Report" \ - --google-drive-folder-id $google_drive_folder_id \ - --env-file ${WORK_DIR}/sast_report_generation_credentials.env + --version ${MONGODB_VERSION} \ + --branch ${MONGODB_RELEASE_BRANCH} \ + --commit-date $commit_datetime \ + --output-path ${MODULE_PATH}/sast_report_${MONGODB_VERSION}.xlsx \ + --upload-file-name "[${MONGODB_VERSION}] MongoDB Server Enterprise SAST Report" \ + --google-drive-folder-id $google_drive_folder_id \ + --env-file ${WORK_DIR}/sast_report_generation_credentials.env diff --git a/evergreen/generate_version.sh b/evergreen/generate_version.sh index 5109c8b8753..60b2e9f7920 100644 --- a/evergreen/generate_version.sh +++ b/evergreen/generate_version.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -9,11 +9,11 @@ set -o verbose setup_mongo_task_generator activate_venv RUST_BACKTRACE=full PATH=$PATH:$HOME:/ ./mongo-task-generator \ - --expansion-file ../expansions.yml \ - --evg-auth-file ./.evergreen.yml \ - --evg-project-file ${evergreen_config_file_path} \ - --generate-sub-tasks-config etc/generate_subtasks_config.yml \ - --s3-test-stats-bucket mongo-test-stats \ - --include-fully-disabled-feature-tests \ - --bazel-suite-configs resmoke_suite_configs.yml \ - $@ + --expansion-file ../expansions.yml \ + --evg-auth-file ./.evergreen.yml \ + --evg-project-file ${evergreen_config_file_path} \ + --generate-sub-tasks-config etc/generate_subtasks_config.yml \ + --s3-test-stats-bucket mongo-test-stats \ + --include-fully-disabled-feature-tests \ + --bazel-suite-configs resmoke_suite_configs.yml \ + $@ diff --git a/evergreen/generate_version_burn_in.sh b/evergreen/generate_version_burn_in.sh index a3f5d65050a..fb717c96e64 100644 --- a/evergreen/generate_version_burn_in.sh +++ b/evergreen/generate_version_burn_in.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -15,12 +15,12 @@ echo "Base patch revision: $base_revision" $python buildscripts/burn_in_tests.py generate-test-membership-map-file-for-ci RUST_BACKTRACE=full PATH=$PATH:$HOME:/ ./mongo-task-generator \ - --expansion-file ../expansions.yml \ - --evg-auth-file ./.evergreen.yml \ - --evg-project-file ${evergreen_config_file_path} \ - --generate-sub-tasks-config etc/generate_subtasks_config.yml \ - --s3-test-stats-bucket mongo-test-stats \ - --include-fully-disabled-feature-tests \ - --burn-in \ - --burn-in-tests-command "python buildscripts/burn_in_tests.py run --origin-rev=$base_revision" \ - $@ + --expansion-file ../expansions.yml \ + --evg-auth-file ./.evergreen.yml \ + --evg-project-file ${evergreen_config_file_path} \ + --generate-sub-tasks-config etc/generate_subtasks_config.yml \ + --s3-test-stats-bucket mongo-test-stats \ + --include-fully-disabled-feature-tests \ + --burn-in \ + --burn-in-tests-command "python buildscripts/burn_in_tests.py run --origin-rev=$base_revision" \ + $@ diff --git a/evergreen/get_all_resmoke_suite_configs.sh b/evergreen/get_all_resmoke_suite_configs.sh index 95e8c60f972..3ce4f690c55 100644 --- a/evergreen/get_all_resmoke_suite_configs.sh +++ b/evergreen/get_all_resmoke_suite_configs.sh @@ -4,7 +4,7 @@ # Usage: # bash get_all_resmoke_suite_configs.sh -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -20,5 +20,5 @@ BAZEL_BINARY=$(bazel_get_binary_path) # str(target.label).replace('@@','') -> the target name, like //buildscripts/resmokeconfig:core_config # f.path for f in target.files.to_list() -> the path to the config file, like bazel-out/k8-fastbuild/bin/buildscripts/resmokeconfig/core.yml ${BAZEL_BINARY} cquery ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \ - --define=MONGO_VERSION=${version} ${patch_compile_flags} "kind(resmoke_config, //...)" \ - --output=starlark --starlark:expr "': '.join([str(target.label).replace('@@','')] + [f.path for f in target.files.to_list()])" > resmoke_suite_configs.yml + --define=MONGO_VERSION=${version} ${patch_compile_flags} "kind(resmoke_config, //...)" \ + --output=starlark --starlark:expr "': '.join([str(target.label).replace('@@','')] + [f.path for f in target.files.to_list()])" >resmoke_suite_configs.yml diff --git a/evergreen/get_bin_and_fcv_versions.sh b/evergreen/get_bin_and_fcv_versions.sh index 0bc91fe4baa..1b60879cbc4 100644 --- a/evergreen/get_bin_and_fcv_versions.sh +++ b/evergreen/get_bin_and_fcv_versions.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src diff --git a/evergreen/get_resmoke_suite_config.sh b/evergreen/get_resmoke_suite_config.sh index e63d56f0c6d..173f5b0d376 100644 --- a/evergreen/get_resmoke_suite_config.sh +++ b/evergreen/get_resmoke_suite_config.sh @@ -6,7 +6,7 @@ # Required environment variables: # * ${suite} - Resmoke bazel target, like //buildscripts/resmokeconfig:core -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -18,4 +18,4 @@ source ./evergreen/bazel_utility_functions.sh BAZEL_BINARY=$(bazel_get_binary_path) echo "suite_config: $(${BAZEL_BINARY} cquery ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \ - --define=MONGO_VERSION=${version} ${patch_compile_flags} ${suite}_config --output files)" > suite_config_expansion.yml + --define=MONGO_VERSION=${version} ${patch_compile_flags} ${suite}_config --output files)" >suite_config_expansion.yml diff --git a/evergreen/hang_analyzer.sh b/evergreen/hang_analyzer.sh index 833104af586..29fbecbdb7a 100644 --- a/evergreen/hang_analyzer.sh +++ b/evergreen/hang_analyzer.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -8,9 +8,9 @@ set -o verbose # Set what processes to look for. For most tasks, we rely on resmoke to figure out its subprocesses # and run the hang analyzer on those. For non-resmoke tasks, we enumerate the process list here. if [[ ${task_name} == *"jepsen"* ]]; then - hang_analyzer_option="-o file -o stdout -p dbtest,java,mongo,mongod,mongos,python,_test" + hang_analyzer_option="-o file -o stdout -p dbtest,java,mongo,mongod,mongos,python,_test" else - hang_analyzer_option="-o file -o stdout -m exact -p python" + hang_analyzer_option="-o file -o stdout -m exact -p python" fi activate_venv @@ -19,5 +19,5 @@ $python buildscripts/resmoke.py hang-analyzer $hang_analyzer_option # Call hang analyzer for tasks that are running remote mongo processes if [ -n "${private_ip_address}" ]; then - $python buildscripts/resmoke.py powercycle remote-hang-analyzer + $python buildscripts/resmoke.py powercycle remote-hang-analyzer fi diff --git a/evergreen/idl_tests_run.sh b/evergreen/idl_tests_run.sh index ec2a8a2f493..bb6e648cdc1 100644 --- a/evergreen/idl_tests_run.sh +++ b/evergreen/idl_tests_run.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src diff --git a/evergreen/jepsen_docker/cleanup.sh b/evergreen/jepsen_docker/cleanup.sh index 9a4c06d8ef5..47e3b13c7d1 100644 --- a/evergreen/jepsen_docker/cleanup.sh +++ b/evergreen/jepsen_docker/cleanup.sh @@ -1,8 +1,8 @@ set -euo pipefail if [ -d jepsen ]; then - echo "Cleanup docker containers" - # docker ps -q fails when no containers are running - sudo docker container kill $(docker ps -q) || true - sudo docker system prune -f + echo "Cleanup docker containers" + # docker ps -q fails when no containers are running + sudo docker container kill $(docker ps -q) || true + sudo docker system prune -f fi diff --git a/evergreen/jepsen_docker/docker-up.sh b/evergreen/jepsen_docker/docker-up.sh index 0a9bba00cc2..0a41d976112 100644 --- a/evergreen/jepsen_docker/docker-up.sh +++ b/evergreen/jepsen_docker/docker-up.sh @@ -1,4 +1,4 @@ set -euo pipefail cd jepsen/docker -./bin/up -n 9 -d 2>&1 > docker.log +./bin/up -n 9 -d 2>&1 >docker.log diff --git a/evergreen/jepsen_docker/list-append.sh b/evergreen/jepsen_docker/list-append.sh index 9896c4699ff..92a6cb570f7 100644 --- a/evergreen/jepsen_docker/list-append.sh +++ b/evergreen/jepsen_docker/list-append.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" # this file does not use set -euo pipefail because we determine test success or @@ -13,9 +13,9 @@ start_time=$(date +%s) # However, to ensure the preservation of all the mongod and mongos logs for each failed test, # we have implemented a for loop to iterate the test 30 times. for i in {1..30}; do - cd jepsen/docker + cd jepsen/docker - sudo docker exec jepsen-control bash --login -c "\ + sudo docker exec jepsen-control bash --login -c "\ cd /jepsen/mongodb && \ lein run test-all -w list-append \ -n n1 -n n2 -n n3 -n n4 -n n5 -n n6 -n n7 -n n8 -n n9 \ @@ -31,47 +31,47 @@ for i in {1..30}; do --nemesis partition \ --test-count 1" | tee jepsen_test_${i}.log - cd ../../ + cd ../../ - # copy files to expected locations for archiving - mkdir -p src/jepsen-mongodb/store/test-index${i} - sudo docker cp jepsen-control:/jepsen/mongodb/store src/jepsen-mongodb/store/test-index${i} - cp jepsen/docker/jepsen_test_${i}.log src/jepsen-mongodb/ - sudo docker cp jepsen-control:/jepsen/mongodb src/jepsen-workdir + # copy files to expected locations for archiving + mkdir -p src/jepsen-mongodb/store/test-index${i} + sudo docker cp jepsen-control:/jepsen/mongodb/store src/jepsen-mongodb/store/test-index${i} + cp jepsen/docker/jepsen_test_${i}.log src/jepsen-mongodb/ + sudo docker cp jepsen-control:/jepsen/mongodb src/jepsen-workdir - # Get the last five lines of the log file like below example - # 1 successes - # 0 unknown - # 0 crashed - # 0 failures - # - last_five_lines=$(tail -n 5 "jepsen/docker/jepsen_test_${i}.log") + # Get the last five lines of the log file like below example + # 1 successes + # 0 unknown + # 0 crashed + # 0 failures + # + last_five_lines=$(tail -n 5 "jepsen/docker/jepsen_test_${i}.log") - # Check if the "1 successes" string is in the last five lines - if echo "$last_five_lines" | grep -q "1 successes"; then - echo "Test is successful, no additional logs will be spared." - else - echo "Test is not successful. Sparing mongod and mongos logs into 'src/jepsen-mongodb/mongodlogs/' and 'src/jepsen-mongodb/mongoslogs/' directories." + # Check if the "1 successes" string is in the last five lines + if echo "$last_five_lines" | grep -q "1 successes"; then + echo "Test is successful, no additional logs will be spared." + else + echo "Test is not successful. Sparing mongod and mongos logs into 'src/jepsen-mongodb/mongodlogs/' and 'src/jepsen-mongodb/mongoslogs/' directories." - # copy mongod logs - mkdir -p src/jepsen-mongodb/mongodlogs/test_${i} - # loop 9 docker containers - for n in {1..9}; do - sudo docker cp jepsen-n${n}:/var/log/mongodb/mongod.log src/jepsen-mongodb/mongodlogs/test_${i}/jepson-n${n}-mongod.log - done - sudo chmod +r src/jepsen-mongodb/mongodlogs/test_${i}/*.log + # copy mongod logs + mkdir -p src/jepsen-mongodb/mongodlogs/test_${i} + # loop 9 docker containers + for n in {1..9}; do + sudo docker cp jepsen-n${n}:/var/log/mongodb/mongod.log src/jepsen-mongodb/mongodlogs/test_${i}/jepson-n${n}-mongod.log + done + sudo chmod +r src/jepsen-mongodb/mongodlogs/test_${i}/*.log - # copy mongos logs - mkdir -p src/jepsen-mongodb/mongoslogs/test_${i} - for n in {1..9}; do - sudo docker cp jepsen-n${n}:/var/log/mongodb/mongos.stdout src/jepsen-mongodb/mongoslogs/test_${i}/jepson-n${n}-mongos.stdout - done - sudo chmod +r src/jepsen-mongodb/mongoslogs/test_${i}/*.stdout - fi + # copy mongos logs + mkdir -p src/jepsen-mongodb/mongoslogs/test_${i} + for n in {1..9}; do + sudo docker cp jepsen-n${n}:/var/log/mongodb/mongos.stdout src/jepsen-mongodb/mongoslogs/test_${i}/jepson-n${n}-mongos.stdout + done + sudo chmod +r src/jepsen-mongodb/mongoslogs/test_${i}/*.stdout + fi done # Merge all jepsen_test_${i}.log into a single file -cat src/jepsen-mongodb/jepsen_test_*.log > src/jepsen-mongodb/jepsen_${task_name}_${execution}.log +cat src/jepsen-mongodb/jepsen_test_*.log >src/jepsen-mongodb/jepsen_${task_name}_${execution}.log end_time=$(date +%s) elapsed_secs=$((end_time - start_time)) @@ -82,8 +82,8 @@ $python buildscripts/jepsen_report.py --start_time=$start_time --end_time=$end_t exit_code=$? if [ -f "jepsen_system_fail.txt" ]; then - mv jepsen_system_fail.txt jepsen-mongodb/jepsen_system_failure_${task_name}_${execution} - exit 0 + mv jepsen_system_fail.txt jepsen-mongodb/jepsen_system_failure_${task_name}_${execution} + exit 0 fi exit $exit_code diff --git a/evergreen/jepsen_docker/setup_config_fuzzer.sh b/evergreen/jepsen_docker/setup_config_fuzzer.sh index 5709dfe590b..834a030c6dd 100644 --- a/evergreen/jepsen_docker/setup_config_fuzzer.sh +++ b/evergreen/jepsen_docker/setup_config_fuzzer.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/../prelude.sh" cd src diff --git a/evergreen/jepsen_report.sh b/evergreen/jepsen_report.sh index d6ec19e9577..da3fd0606a8 100644 --- a/evergreen/jepsen_report.sh +++ b/evergreen/jepsen_report.sh @@ -4,22 +4,22 @@ grep -q "Everything looks good" jepsen_${task_name}_${execution}.log grep_exit_code=$? if [ $grep_exit_code -eq 0 ]; then - status='"pass"' - failures=0 - final_exit_code=0 + status='"pass"' + failures=0 + final_exit_code=0 else - grep -q "Analysis invalid" jepsen_${task_name}_${execution}.log - grep_exit_code=$? - if [ $grep_exit_code -eq 0 ]; then - status='"fail"' - failures=1 - final_exit_code=1 - else - # If the failure is due to setup, then this is considered a system failure. - echo $grep_exit_code > jepsen_system_failure_${task_name}_${execution} - exit 0 - fi + grep -q "Analysis invalid" jepsen_${task_name}_${execution}.log + grep_exit_code=$? + if [ $grep_exit_code -eq 0 ]; then + status='"fail"' + failures=1 + final_exit_code=1 + else + # If the failure is due to setup, then this is considered a system failure. + echo $grep_exit_code >jepsen_system_failure_${task_name}_${execution} + exit 0 + fi fi # Create report.json -echo "{\"failures\": $failures, \"results\": [{\"status\": $status, \"exit_code\": $final_exit_code, \"test_file\": \"${task_name}\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > ../report.json +echo "{\"failures\": $failures, \"results\": [{\"status\": $status, \"exit_code\": $final_exit_code, \"test_file\": \"${task_name}\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" >../report.json exit $final_exit_code diff --git a/evergreen/jepsen_test_fail.sh b/evergreen/jepsen_test_fail.sh index 0544758d66f..e6c9fd64c5d 100644 --- a/evergreen/jepsen_test_fail.sh +++ b/evergreen/jepsen_test_fail.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src/jepsen-mongodb @@ -6,5 +6,5 @@ cd src/jepsen-mongodb set -o verbose # Jepsen system failure if file exists. if [ -f jepsen_system_failure_${task_name}_${execution} ]; then - exit $(cat jepsen_system_failure_${task_name}_${execution}) + exit $(cat jepsen_system_failure_${task_name}_${execution}) fi diff --git a/evergreen/jepsen_test_run.sh b/evergreen/jepsen_test_run.sh index 719b4a358d4..42130209e2c 100644 --- a/evergreen/jepsen_test_run.sh +++ b/evergreen/jepsen_test_run.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src/jepsen-mongodb @@ -22,22 +22,22 @@ start_time=$(date +%s) # will need to adopt buildscripts/jepsen_report.py to support the log output # of older-style Jepsen tests lein run test --test ${jepsen_test_name} \ - --mongodb-dir ../ \ - --working-dir ${workdir}/src/jepsen-workdir \ - --clock-skew faketime \ - --libfaketime-path ${workdir}/src/libfaketime/build/libfaketime.so.1 \ - --virtualization none \ - --nodes-file ../nodes.txt \ - ${mongod_conf} \ - ${jepsen_key_time_limit} \ - ${jepsen_protocol_version} \ - ${jepsen_read_concern} \ - ${jepsen_read_with_find_and_modify} \ - ${jepsen_storage_engine} \ - ${jepsen_time_limit} \ - ${jepsen_write_concern} \ - 2>&1 \ - | tee jepsen_${task_name}_${execution}.log + --mongodb-dir ../ \ + --working-dir ${workdir}/src/jepsen-workdir \ + --clock-skew faketime \ + --libfaketime-path ${workdir}/src/libfaketime/build/libfaketime.so.1 \ + --virtualization none \ + --nodes-file ../nodes.txt \ + ${mongod_conf} \ + ${jepsen_key_time_limit} \ + ${jepsen_protocol_version} \ + ${jepsen_read_concern} \ + ${jepsen_read_with_find_and_modify} \ + ${jepsen_storage_engine} \ + ${jepsen_time_limit} \ + ${jepsen_write_concern} \ + 2>&1 | + tee jepsen_${task_name}_${execution}.log end_time=$(date +%s) elapsed_secs=$((end_time - start_time)) . ../evergreen/jepsen_report.sh diff --git a/evergreen/jstestfuzz_minimize.sh b/evergreen/jstestfuzz_minimize.sh index e1a3f54a1a4..ea358a66a23 100644 --- a/evergreen/jstestfuzz_minimize.sh +++ b/evergreen/jstestfuzz_minimize.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src/jstestfuzz @@ -7,5 +7,5 @@ set -o errexit set -o verbose if [ -f "../minimizer-outputs.json" ]; then - eval ./src/scripts/npm_run.sh ${npm_command} -- -j "../minimizer-outputs.json" + eval ./src/scripts/npm_run.sh ${npm_command} -- -j "../minimizer-outputs.json" fi diff --git a/evergreen/jstestfuzz_run.sh b/evergreen/jstestfuzz_run.sh index adb1d4e7bab..b810ec5e7fa 100644 --- a/evergreen/jstestfuzz_run.sh +++ b/evergreen/jstestfuzz_run.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" set -evo pipefail @@ -11,8 +11,8 @@ vars="${jstestfuzz_vars}" # # 'jstestfuzz_gitrev' and remove it from 'vars'. echo "${vars}" | grep -q -- '--jstestfuzzGitRev[ \t]\+[A-Za-z0-9]\+' && RET=0 || RET=$? if [ $RET -eq 0 ]; then - jstestfuzz_gitrev=$(echo "${vars}" | sed -e 's/.*--jstestfuzzGitRev[ \t]\+\([A-Za-z0-9]\+\).*/\1/') - vars=$(echo "${vars}" | sed -e 's/\(.*\)--jstestfuzzGitRev[ \t]\+[A-Za-z0-9]\+\(.*\)/\1\2/') + jstestfuzz_gitrev=$(echo "${vars}" | sed -e 's/.*--jstestfuzzGitRev[ \t]\+\([A-Za-z0-9]\+\).*/\1/') + vars=$(echo "${vars}" | sed -e 's/\(.*\)--jstestfuzzGitRev[ \t]\+[A-Za-z0-9]\+\(.*\)/\1\2/') fi # If the "--metaSeed" option is present in 'vars', copy the option's value into 'meta_seed' @@ -21,49 +21,49 @@ fi generated_seed_flag="" echo "${vars}" | grep -q -- '--metaSeed[ \t]\+[0-9]\+' && RET=0 || RET=$? if [ $RET -eq 0 ]; then - # Throw an error if the "--seed" option and the "--metaSeed" option are both present. - echo "${vars}" | grep -q -- '--seed[ \t]\+[0-9]\+' && RET=0 || RET=$? - if [ $RET -eq 0 ]; then - echo "Cannot use the --seed option and the --metaSeed option together" - exit 1 - fi + # Throw an error if the "--seed" option and the "--metaSeed" option are both present. + echo "${vars}" | grep -q -- '--seed[ \t]\+[0-9]\+' && RET=0 || RET=$? + if [ $RET -eq 0 ]; then + echo "Cannot use the --seed option and the --metaSeed option together" + exit 1 + fi - # Store the meta seed value into 'meta_seed' and remove the "--metaSeed" option from 'vars'. - meta_seed=$(echo "${vars}" | sed -e 's/.*--metaSeed[ \t]\+\([0-9]\+\).*/\1/') - vars=$(echo "${vars}" | sed -e 's/\(.*\)--metaSeed[ \t]\+[0-9]\+\(.*\)/\1\2/') + # Store the meta seed value into 'meta_seed' and remove the "--metaSeed" option from 'vars'. + meta_seed=$(echo "${vars}" | sed -e 's/.*--metaSeed[ \t]\+\([0-9]\+\).*/\1/') + vars=$(echo "${vars}" | sed -e 's/\(.*\)--metaSeed[ \t]\+[0-9]\+\(.*\)/\1\2/') - # If 'task_name' matches the pattern that we use for generated task names, extract the number - # from 'task_name' and generate a seed using this number and 'meta_seed' together. Otherwise, - # just use 'meta_seed' itself for the seed. - echo "${task_name}" | grep -q '_[0-9]\+\(-.*\)\?$' && RET=0 || RET=$? - if [ $RET -eq 0 ]; then - task_num=$(echo "${task_name}" | sed -e 's/.*_\([0-9]\+\)\(-.*\)\?$/\1/') - generated_seed=$((meta_seed + 971 * (task_num + 1))) - generated_seed_flag="--seed ${generated_seed}" - else - generated_seed_flag="--seed ${meta_seed}" - fi + # If 'task_name' matches the pattern that we use for generated task names, extract the number + # from 'task_name' and generate a seed using this number and 'meta_seed' together. Otherwise, + # just use 'meta_seed' itself for the seed. + echo "${task_name}" | grep -q '_[0-9]\+\(-.*\)\?$' && RET=0 || RET=$? + if [ $RET -eq 0 ]; then + task_num=$(echo "${task_name}" | sed -e 's/.*_\([0-9]\+\)\(-.*\)\?$/\1/') + generated_seed=$((meta_seed + 971 * (task_num + 1))) + generated_seed_flag="--seed ${generated_seed}" + else + generated_seed_flag="--seed ${meta_seed}" + fi fi in_patch_build_flag="" if [[ "${is_patch}" = "true" ]]; then - case "${npm_command}" in - agg-fuzzer | query-fuzzer) - in_patch_build_flag="--inPatchBuild" - ;; - esac + case "${npm_command}" in + agg-fuzzer | query-fuzzer) + in_patch_build_flag="--inPatchBuild" + ;; + esac fi # TODO(DEVPROD-10137): Remove this conditional logic once `--useEsModule` is a top-level supported flag of jstestfuzz. maybe_use_es_modules="" if [[ "${npm_command}" != "jstestfuzz" ]]; then - maybe_use_es_modules="--useEsModules" + maybe_use_es_modules="--useEsModules" fi mkdir -p ./src/jstestfuzz/out $CONTAINER_RUNTIME run --rm \ - -v "./src/jstestfuzz/out:/app/out" \ - -v "./src/jstests:/app/jstests" \ - 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:${jstestfuzz_gitrev:-latest} \ - npm run-script ${npm_command} -- ${vars} ${generated_seed_flag} ${in_patch_build_flag} ${maybe_use_es_modules} --branch ${branch_name} + -v "./src/jstestfuzz/out:/app/out" \ + -v "./src/jstests:/app/jstests" \ + 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:${jstestfuzz_gitrev:-latest} \ + npm run-script ${npm_command} -- ${vars} ${generated_seed_flag} ${in_patch_build_flag} ${maybe_use_es_modules} --branch ${branch_name} diff --git a/evergreen/jstestfuzz_setup.sh b/evergreen/jstestfuzz_setup.sh index fa6e37e2490..88766487ce7 100644 --- a/evergreen/jstestfuzz_setup.sh +++ b/evergreen/jstestfuzz_setup.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -6,18 +6,18 @@ cd src set -o errexit set -o verbose -if command -v docker > /dev/null 2>&1; then - echo "Docker is installed, using docker" - CONTAINER_RUNTIME=docker +if command -v docker >/dev/null 2>&1; then + echo "Docker is installed, using docker" + CONTAINER_RUNTIME=docker -elif command -v podman > /dev/null 2>&1; then - echo "Podman is installed, using podman" - CONTAINER_RUNTIME=podman +elif command -v podman >/dev/null 2>&1; then + echo "Podman is installed, using podman" + CONTAINER_RUNTIME=podman else - echo "Neither Docker nor Podman is installed. Please install one of them." - exit 1 + echo "Neither Docker nor Podman is installed. Please install one of them." + exit 1 fi -echo "CONTAINER_RUNTIME: ${CONTAINER_RUNTIME}" >> expansions.yml +echo "CONTAINER_RUNTIME: ${CONTAINER_RUNTIME}" >>expansions.yml aws ecr get-login-password --region us-east-1 | $CONTAINER_RUNTIME login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com diff --git a/evergreen/kill_processes.sh b/evergreen/kill_processes.sh index fc2f732b34f..50c38b81d1c 100755 --- a/evergreen/kill_processes.sh +++ b/evergreen/kill_processes.sh @@ -3,52 +3,52 @@ process_kill_list="(^cl\.exe$|java|lein|lldb|mongo|python|_test$|_test\.exe$)" process_exclude_list="(main|tuned|evergreen)" if [ "Windows_NT" = "$OS" ]; then - # Get the list of Windows tasks (tasklist list format): - # - Transpose the Image Name and PID - # - The first column has the process ID - # - The second column (and beyond) has task name - # - Grep for the task names of interest while ignoring any names that are in the exclude list + # Get the list of Windows tasks (tasklist list format): + # - Transpose the Image Name and PID + # - The first column has the process ID + # - The second column (and beyond) has task name + # - Grep for the task names of interest while ignoring any names that are in the exclude list - processes=$(tasklist /fo:csv | awk -F'","' '{x=$1; gsub("\"","",x); print $2, x}' | grep -iE "$process_kill_list" | grep -ivE "$process_exclude_list") + processes=$(tasklist /fo:csv | awk -F'","' '{x=$1; gsub("\"","",x); print $2, x}' | grep -iE "$process_kill_list" | grep -ivE "$process_exclude_list") - # Kill the Windows process by process ID with force (/f) - kill_process() { - pid=$(echo $1 | cut -f1 -d ' ') - echo "Killing process $1" - taskkill /pid "$pid" /f - } + # Kill the Windows process by process ID with force (/f) + kill_process() { + pid=$(echo $1 | cut -f1 -d ' ') + echo "Killing process $1" + taskkill /pid "$pid" /f + } else - # Get the list of Unix tasks (pgrep full & long): - # - Grep for the task names of interest while ignoring any names that are in the exclude list - # - The first column has the process ID - # - The second column (and beyond) has task name + # Get the list of Unix tasks (pgrep full & long): + # - Grep for the task names of interest while ignoring any names that are in the exclude list + # - The first column has the process ID + # - The second column (and beyond) has task name - # There are 2 "styles" of pgrep, figure out which one works. - # Due to https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1501916 - # we cannot rely on the return status ($?) to detect if the option is supported. - pgrep -f --list-full ".*" 2>&1 | grep -qE "(illegal|invalid|unrecognized) option" - if [ $? -ne 0 ]; then - pgrep_list=$(pgrep -f --list-full "$process_kill_list") - else - pgrep_list=$(pgrep -f -l "$process_kill_list") - fi + # There are 2 "styles" of pgrep, figure out which one works. + # Due to https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1501916 + # we cannot rely on the return status ($?) to detect if the option is supported. + pgrep -f --list-full ".*" 2>&1 | grep -qE "(illegal|invalid|unrecognized) option" + if [ $? -ne 0 ]; then + pgrep_list=$(pgrep -f --list-full "$process_kill_list") + else + pgrep_list=$(pgrep -f -l "$process_kill_list") + fi - # Since a process name might have a CR or LF in it, we need to delete any lines from - # pgrep which do not start with space(s) and 1 digit and trim any leading spaces. - processes=$(echo "$pgrep_list" | grep -ivE "$process_exclude_list" | sed -e '/^ *[0-9]/!d; s/^ *//; s/[[:cntrl:]]//g;') + # Since a process name might have a CR or LF in it, we need to delete any lines from + # pgrep which do not start with space(s) and 1 digit and trim any leading spaces. + processes=$(echo "$pgrep_list" | grep -ivE "$process_exclude_list" | sed -e '/^ *[0-9]/!d; s/^ *//; s/[[:cntrl:]]//g;') - # Kill the Unix process ID with signal KILL (9) - kill_process() { - pid=$(echo $1 | cut -f1 -d ' ') - echo "Killing process $1" - kill -9 $pid - } + # Kill the Unix process ID with signal KILL (9) + kill_process() { + pid=$(echo $1 | cut -f1 -d ' ') + echo "Killing process $1" + kill -9 $pid + } fi # Since a full process name can have spaces, the IFS (internal field separator) # should not include a space, just a LF & CR IFS=$(printf "\n\r") for process in $processes; do - kill_process "$process" + kill_process "$process" done exit 0 diff --git a/evergreen/lint_fuzzer_sanity_all.sh b/evergreen/lint_fuzzer_sanity_all.sh index d1701af3c20..1608c1611ec 100644 --- a/evergreen/lint_fuzzer_sanity_all.sh +++ b/evergreen/lint_fuzzer_sanity_all.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -13,9 +13,9 @@ CONTAINER_INPUT_DIR="/app" podman pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest # Run parse-jsfiles on 50 files at a time with 32 processes in parallel. # Skip javascript files in third_party directory -find "jstests" "src/mongo/db/modules/enterprise" -path "$PWD/jstests/third_party" -prune -o -name "*.js" -print \ - | xargs -P 32 -L 50 $CONTAINER_RUNTIME run -v $LOCAL_INPUT_DIR/jstests:$CONTAINER_INPUT_DIR/jstests -v $LOCAL_INPUT_DIR/src/mongo/db/modules/enterprise:$CONTAINER_INPUT_DIR/src/mongo/db/modules/enterprise 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest npm run-script parse-jsfiles -- 2>&1 \ - | tee lint_fuzzer_sanity.log +find "jstests" "src/mongo/db/modules/enterprise" -path "$PWD/jstests/third_party" -prune -o -name "*.js" -print | + xargs -P 32 -L 50 $CONTAINER_RUNTIME run -v $LOCAL_INPUT_DIR/jstests:$CONTAINER_INPUT_DIR/jstests -v $LOCAL_INPUT_DIR/src/mongo/db/modules/enterprise:$CONTAINER_INPUT_DIR/src/mongo/db/modules/enterprise 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest npm run-script parse-jsfiles -- 2>&1 | + tee lint_fuzzer_sanity.log exit_code=$? activate_venv diff --git a/evergreen/lint_fuzzer_sanity_patch.sh b/evergreen/lint_fuzzer_sanity_patch.sh index abca8ef8fca..1b98342cf54 100644 --- a/evergreen/lint_fuzzer_sanity_patch.sh +++ b/evergreen/lint_fuzzer_sanity_patch.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" set -o errexit diff --git a/evergreen/lint_shellscripts.sh b/evergreen/lint_shellscripts.sh deleted file mode 100644 index 615f288c308..00000000000 --- a/evergreen/lint_shellscripts.sh +++ /dev/null @@ -1,14 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -set -o pipefail - -cd src - -PATH="/opt/shfmt/v3.2.4/bin:$PATH" -./buildscripts/shellscripts-linters.sh | tee shellscripts.log -exit_code=$? - -activate_venv -$python ./buildscripts/simple_report.py --test-name shfmt --log-file shellscripts.log --exit-code $exit_code -exit $exit_code diff --git a/evergreen/lint_yaml.sh b/evergreen/lint_yaml.sh index 0610ce1db6c..a5036987baa 100755 --- a/evergreen/lint_yaml.sh +++ b/evergreen/lint_yaml.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" set -o pipefail diff --git a/evergreen/local_client_logs_tar.sh b/evergreen/local_client_logs_tar.sh index dc8dc391c46..4a2eed5f7e4 100644 --- a/evergreen/local_client_logs_tar.sh +++ b/evergreen/local_client_logs_tar.sh @@ -1,9 +1,9 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src -client_logs=$(ls crud*.log fsm*.log 2> /dev/null) +client_logs=$(ls crud*.log fsm*.log 2>/dev/null) if [ ! -z "$client_logs" ]; then - ${tar} czf client-logs.tgz $client_logs + ${tar} czf client-logs.tgz $client_logs fi diff --git a/evergreen/modify_debug_symbols.sh b/evergreen/modify_debug_symbols.sh index 134bb411c15..a363eeb87d2 100644 --- a/evergreen/modify_debug_symbols.sh +++ b/evergreen/modify_debug_symbols.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" set -o errexit diff --git a/evergreen/monitor_build_status_run.sh b/evergreen/monitor_build_status_run.sh index 313796fe1d9..dc22e8360af 100755 --- a/evergreen/monitor_build_status_run.sh +++ b/evergreen/monitor_build_status_run.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" set -o errexit @@ -9,7 +9,7 @@ activate_venv command_invocation="$python buildscripts/monitor_build_status/cli.py" if [ "${is_patch}" != "true" ]; then - command_invocation="$command_invocation --notify" + command_invocation="$command_invocation --notify" fi echo "Verbatim monitor_build_status invocation: ${command_invocation}" diff --git a/evergreen/move_npm_logs.sh b/evergreen/move_npm_logs.sh index 850d0baa861..7bcf49ad6f6 100644 --- a/evergreen/move_npm_logs.sh +++ b/evergreen/move_npm_logs.sh @@ -1,10 +1,10 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" set -o errexit set -o verbose if [ -d "${HOME}/.npm/_logs" ] && [ -n "$(ls -A "${HOME}/.npm/_logs")" ]; then - cp -r "${HOME}"/.npm/_logs "${workdir}" - rm -rf "${HOME}"/.npm/_logs/* + cp -r "${HOME}"/.npm/_logs "${workdir}" + rm -rf "${HOME}"/.npm/_logs/* fi diff --git a/evergreen/multiversion_exclude_tags_generate.sh b/evergreen/multiversion_exclude_tags_generate.sh index 147a9a531bc..2e6dd7ffef5 100644 --- a/evergreen/multiversion_exclude_tags_generate.sh +++ b/evergreen/multiversion_exclude_tags_generate.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -10,14 +10,14 @@ activate_venv PATH="$PATH:/data/multiversion" if [[ "${require_multiversion_setup}" = "true" && -n "${multiversion_exclude_tags_version}" ]]; then - exclude_tags_file_path_arg="" - if [ ! -d generated_resmoke_config ]; then - exclude_tags_file_path_arg="--excludeTagsFilePath=multiversion_exclude_tags.yml" - fi + exclude_tags_file_path_arg="" + if [ ! -d generated_resmoke_config ]; then + exclude_tags_file_path_arg="--excludeTagsFilePath=multiversion_exclude_tags.yml" + fi - if [ -n "${multiversion_link_dir}" ]; then - multiversion_dir_arg="--multiversionDir=${multiversion_link_dir}" - fi + if [ -n "${multiversion_link_dir}" ]; then + multiversion_dir_arg="--multiversionDir=${multiversion_link_dir}" + fi - eval $python buildscripts/resmoke.py generate-multiversion-exclude-tags --oldBinVersion="${multiversion_exclude_tags_version}" "$exclude_tags_file_path_arg" "$multiversion_dir_arg" + eval $python buildscripts/resmoke.py generate-multiversion-exclude-tags --oldBinVersion="${multiversion_exclude_tags_version}" "$exclude_tags_file_path_arg" "$multiversion_dir_arg" fi diff --git a/evergreen/multiversion_selection.sh b/evergreen/multiversion_selection.sh index b2354b9f1dc..4ad0ef59613 100644 --- a/evergreen/multiversion_selection.sh +++ b/evergreen/multiversion_selection.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -21,20 +21,20 @@ last_lts_arg="last-lts" last_continuous_arg="last-continuous" if [[ -n "${last_lts_evg_version_id}" ]]; then - last_lts_arg="${last_lts_evg_version_id}=last-lts" + last_lts_arg="${last_lts_evg_version_id}=last-lts" fi if [[ -n "${last_continuous_evg_version_id}" ]]; then - last_continuous_arg="${last_continuous_evg_version_id}=last-continuous" + last_continuous_arg="${last_continuous_evg_version_id}=last-continuous" fi # Download last-continuous version separately on future-git-tag variant if [[ -n "${multiversion_last_continuous_variant}" ]]; then - last_continuous_arg="" + last_continuous_arg="" fi if [ ! -z "${multiversion_platform_50_or_later}" ]; then - platform="${multiversion_platform_50_or_later}" + platform="${multiversion_platform_50_or_later}" fi base_command="db-contrib-tool setup-repro-env -sb" @@ -55,24 +55,24 @@ echo "Verbatim db-contrib-tool invocation: ${remote_invocation}" local_invocation="${local_invocation} && ${base_command} ${local_args}" echo "Local db-contrib-tool invocation: ${local_invocation}" -echo "${local_invocation}" > local-db-contrib-tool-invocation.txt +echo "${local_invocation}" >local-db-contrib-tool-invocation.txt # Download last-continuous version from a dedicated variant on future-git-tag variant if [[ -n "${multiversion_last_continuous_variant}" ]]; then - last_continuous_arg="${version_id}=last-continuous" + last_continuous_arg="${version_id}=last-continuous" - if [[ -n "${last_continuous_evg_version_id}" ]]; then - last_continuous_arg="${last_continuous_evg_version_id}=last-continuous" - fi + if [[ -n "${last_continuous_evg_version_id}" ]]; then + last_continuous_arg="${last_continuous_evg_version_id}=last-continuous" + fi - future_git_tag_args="--installDir /data/install \ + future_git_tag_args="--installDir /data/install \ --linkDir /data/multiversion \ --variant ${multiversion_last_continuous_variant} \ --evgVersionsFile multiversion-downloads.json \ --debug \ ${last_continuous_arg}" - remote_invocation="${base_command} ${future_git_tag_args}" - eval "${remote_invocation}" - echo "Verbatim db-contrib-tool invocation: ${remote_invocation}" + remote_invocation="${base_command} ${future_git_tag_args}" + eval "${remote_invocation}" + echo "Verbatim db-contrib-tool invocation: ${remote_invocation}" fi diff --git a/evergreen/multiversion_setup.sh b/evergreen/multiversion_setup.sh index c8f06eb2f68..47b4b5148d6 100644 --- a/evergreen/multiversion_setup.sh +++ b/evergreen/multiversion_setup.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src diff --git a/evergreen/package.sh b/evergreen/package.sh index 91bc2ec3df4..719eb0e0d8c 100755 --- a/evergreen/package.sh +++ b/evergreen/package.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -9,56 +9,56 @@ set -o verbose activate_venv valid_mongocryptd_variants=( - "enterprise-amazon2" - "enterprise-amazon2-arm64" - "enterprise-amazon2-streams" - "enterprise-amazon2-streams-arm64" - "enterprise-amazon2023" - "enterprise-amazon2023-arm64" - "enterprise-amazon2023-arm64-grav4" - "enterprise-amazon2023-streams" - "enterprise-debian12-64" - "enterprise-linux-64-amazon-ami" - "enterprise-macos" - "enterprise-macos-arm64" - "enterprise-rhel-81-ppc64le" - "enterprise-rhel-8-64-bit" - "enterprise-rhel-8-64-bit-coverage" - "enterprise-rhel-8-64-bit-suggested" - "enterprise-rhel-8-arm64" - "enterprise-rhel-83-s390x" - "enterprise-rhel-9-ppc64le" - "enterprise-rhel-9-s390x" - "enterprise-rhel-90-64-bit" - "enterprise-rhel-90-arm64" - "enterprise-rhel-93-64-bit" - "enterprise-rhel-93-arm64" - "enterprise-suse15-64" - "enterprise-ubuntu2004-arm64" - "enterprise-ubuntu2204-arm64" - "enterprise-ubuntu2204-jepsen" - "enterprise-ubuntu2404" - "enterprise-ubuntu2404-arm64" - "enterprise-ubuntu2004-64" - "enterprise-ubuntu2204-64" - "enterprise-windows" - "enterprise-windows-debug-unoptimized" - "enterprise-windows-inmem" - "enterprise-windows-wtdevelop" + "enterprise-amazon2" + "enterprise-amazon2-arm64" + "enterprise-amazon2-streams" + "enterprise-amazon2-streams-arm64" + "enterprise-amazon2023" + "enterprise-amazon2023-arm64" + "enterprise-amazon2023-arm64-grav4" + "enterprise-amazon2023-streams" + "enterprise-debian12-64" + "enterprise-linux-64-amazon-ami" + "enterprise-macos" + "enterprise-macos-arm64" + "enterprise-rhel-81-ppc64le" + "enterprise-rhel-8-64-bit" + "enterprise-rhel-8-64-bit-coverage" + "enterprise-rhel-8-64-bit-suggested" + "enterprise-rhel-8-arm64" + "enterprise-rhel-83-s390x" + "enterprise-rhel-9-ppc64le" + "enterprise-rhel-9-s390x" + "enterprise-rhel-90-64-bit" + "enterprise-rhel-90-arm64" + "enterprise-rhel-93-64-bit" + "enterprise-rhel-93-arm64" + "enterprise-suse15-64" + "enterprise-ubuntu2004-arm64" + "enterprise-ubuntu2204-arm64" + "enterprise-ubuntu2204-jepsen" + "enterprise-ubuntu2404" + "enterprise-ubuntu2404-arm64" + "enterprise-ubuntu2004-64" + "enterprise-ubuntu2204-64" + "enterprise-windows" + "enterprise-windows-debug-unoptimized" + "enterprise-windows-inmem" + "enterprise-windows-wtdevelop" ) if [ $(find . -name mongocryptd${exe} | wc -l) -ge 1 ]; then - echo "Validating that ${build_variant} is a known enterprise task for mongocryptd" - for valid_mongocryptd_variant in "${valid_mongocryptd_variants[@]}"; do - if [[ "$build_variant" == "$valid_mongocryptd_variant" ]] || "$build_variant" == "${valid_mongocryptd_variant}-toolchain-v5"; then - exit 0 - fi - done - echo "ERROR: ${build_variant} is not a known enterprise task for mongocryptd" - exit 1 - # TODO(SERVER-100860): Fix validate_mongocryptd.py and re-enable it instead of the loop above - # eval PATH=$PATH:$HOME $python ./buildscripts/validate_mongocryptd.py --variant "${build_variant}" etc/evergreen_yml_components/tasks/compile_tasks.yml + echo "Validating that ${build_variant} is a known enterprise task for mongocryptd" + for valid_mongocryptd_variant in "${valid_mongocryptd_variants[@]}"; do + if [[ "$build_variant" == "$valid_mongocryptd_variant" ]] || "$build_variant" == "${valid_mongocryptd_variant}-toolchain-v5"; then + exit 0 + fi + done + echo "ERROR: ${build_variant} is not a known enterprise task for mongocryptd" + exit 1 + # TODO(SERVER-100860): Fix validate_mongocryptd.py and re-enable it instead of the loop above + # eval PATH=$PATH:$HOME $python ./buildscripts/validate_mongocryptd.py --variant "${build_variant}" etc/evergreen_yml_components/tasks/compile_tasks.yml else - echo "Skipping validation of ${build_variant} as the repository does not have a mongocryptd binary" - exit 0 + echo "Skipping validation of ${build_variant} as the repository does not have a mongocryptd binary" + exit 0 fi diff --git a/evergreen/packager.py_run.sh b/evergreen/packager.py_run.sh index ea71996e701..9545f8f02c2 100644 --- a/evergreen/packager.py_run.sh +++ b/evergreen/packager.py_run.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -8,15 +8,15 @@ set -o verbose activate_venv if [ "${has_packages}" != "true" ]; then - exit 0 - # TODO: SERVER-80443 add back validation - # echo "Error: packager run when has_packages is not set to true, please remove the package task from this variant (or variant task group) or set has_packages to true if this variant is intended to run the packager." - # exit 1 + exit 0 + # TODO: SERVER-80443 add back validation + # echo "Error: packager run when has_packages is not set to true, please remove the package task from this variant (or variant task group) or set has_packages to true if this variant is intended to run the packager." + # exit 1 fi if [ -z ${packager_script+x} ]; then - echo "Error: packager run when packager_script is not set, please remove the package task from this variant (or variant task group) or set packager_script if this variant is intended to run the packager." - exit 1 + echo "Error: packager run when packager_script is not set, please remove the package task from this variant (or variant task group) or set packager_script if this variant is intended to run the packager." + exit 1 fi cd buildscripts diff --git a/evergreen/packager_crypt_py_run.sh b/evergreen/packager_crypt_py_run.sh index e146dafaaa1..152fb898b65 100644 --- a/evergreen/packager_crypt_py_run.sh +++ b/evergreen/packager_crypt_py_run.sh @@ -7,20 +7,20 @@ activate_venv [[ "${has_packages}" != "true" ]] && exit 0 if [[ -z "${packager_script+x}" ]]; then - echo "Error: packager run when packager_script is not set, please remove the package task from this variant (or variant task group) or set packager_script if this variant is intended to run the packager." - exit 1 + echo "Error: packager run when packager_script is not set, please remove the package task from this variant (or variant task group) or set packager_script if this variant is intended to run the packager." + exit 1 fi pushd "src/buildscripts" >&/dev/null trap 'popd >& /dev/null' EXIT $python \ - ${packager_script} \ - --prefix $(pwd)/.. \ - --distros \ - ${packager_distro} \ - --crypt_spec \ - --tarball $(pwd)/../mongo_crypt_shared_v1-${version}.${ext:-tgz} \ - -s ${version} \ - -m HEAD \ - -a ${packager_arch} + ${packager_script} \ + --prefix $(pwd)/.. \ + --distros \ + ${packager_distro} \ + --crypt_spec \ + --tarball $(pwd)/../mongo_crypt_shared_v1-${version}.${ext:-tgz} \ + -s ${version} \ + -m HEAD \ + -a ${packager_arch} diff --git a/evergreen/packages_publish.sh b/evergreen/packages_publish.sh index 6b2ecca5755..022682a01be 100644 --- a/evergreen/packages_publish.sh +++ b/evergreen/packages_publish.sh @@ -1,16 +1,16 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" packagesfile=packages.tgz -curl https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz >> $packagesfile +curl https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz >>$packagesfile podman run \ - -v $(pwd):$(pwd) \ - -w $(pwd) \ - --env-host \ - ${UPLOAD_LOCK_IMAGE} \ - -key=${version_id}/${build_id}/packages/${packagesfile} -tag=task-id=${EVERGREEN_TASK_ID} ${packagesfile} + -v $(pwd):$(pwd) \ + -w $(pwd) \ + --env-host \ + ${UPLOAD_LOCK_IMAGE} \ + -key=${version_id}/${build_id}/packages/${packagesfile} -tag=task-id=${EVERGREEN_TASK_ID} ${packagesfile} cd src diff --git a/evergreen/papertrail_generate_expansions.sh b/evergreen/papertrail_generate_expansions.sh index 20c945898a2..1215aea7d9b 100755 --- a/evergreen/papertrail_generate_expansions.sh +++ b/evergreen/papertrail_generate_expansions.sh @@ -6,10 +6,10 @@ set -o verbose version=$(git describe --tags --always --dirty) if [ ${IS_RELEASE} = 'true' ]; then - version="${version#r}" + version="${version#r}" fi -cat << EOT > papertrail-expansions.yml +cat <papertrail-expansions.yml release_version: "$version" EOT cat papertrail-expansions.yml diff --git a/evergreen/perf-submission.sh b/evergreen/perf-submission.sh index 351f6da91dc..55c2e885b8c 100644 --- a/evergreen/perf-submission.sh +++ b/evergreen/perf-submission.sh @@ -4,25 +4,25 @@ set -eu if [ "${requester}" == "commit" ]; then - is_mainline=true + is_mainline=true else - is_mainline=false + is_mainline=false fi # Parse the username out of the order_id. Patches append the username. The Signal Processing Service (SPS) endpoint does not need the other information. parsed_order_id=$(echo "${revision_order_id}" | awk -F'_' '{print $NF}') # Submit the performance data to the SPS endpoint response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X 'POST' \ - "https://performance-monitoring-api.corp.mongodb.com/raw_perf_results/cedar_report?project=${project_id}&version=${version_id}&variant=${build_variant}&order=$parsed_order_id&task_name=${task_name}&task_id=${task_id}&execution=${execution}&mainline=$is_mainline" \ - -H 'accept: application/json' \ - -H 'Content-Type: application/json' \ - -d @src/cedar_report.json) + "https://performance-monitoring-api.corp.mongodb.com/raw_perf_results/cedar_report?project=${project_id}&version=${version_id}&variant=${build_variant}&order=$parsed_order_id&task_name=${task_name}&task_id=${task_id}&execution=${execution}&mainline=$is_mainline" \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d @src/cedar_report.json) http_status=$(echo "$response" | grep "HTTP_STATUS" | awk -F':' '{print $2}') response_body=$(echo "$response" | sed '/HTTP_STATUS/d') # We want to throw an error if the data was not successfully submitted if [ "$http_status" -ne 200 ]; then - echo "Error: Received HTTP status $http_status" - echo "Response Body: $response_body" - exit 1 + echo "Error: Received HTTP status $http_status" + echo "Response Body: $response_body" + exit 1 fi echo "Response Body: $response_body" echo "HTTP Status: $http_status" diff --git a/evergreen/powercycle_check_host.sh b/evergreen/powercycle_check_host.sh index c0ad01890a4..cdeeca8f8d9 100644 --- a/evergreen/powercycle_check_host.sh +++ b/evergreen/powercycle_check_host.sh @@ -1,15 +1,15 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" if [ "Windows_NT" = "$OS" ]; then - user=Administrator + user=Administrator else - user=$USER + user=$USER fi -hostname=$(tr -d '"[]{}' < src/hosts.yml | cut -d , -f 1 | awk -F : '{print $2}') +hostname=$(tr -d '"[]{}' > src/powercycle_ip_address.yml +echo "private_ip_address: $hostname" >>src/powercycle_ip_address.yml echo $hostname echo $user @@ -19,16 +19,16 @@ connection_attempts=60 # Check for remote connectivity while ! ssh \ - -i ${private_key_file} \ - -o ConnectTimeout=10 \ - -o ForwardAgent=yes \ - -o IdentitiesOnly=yes \ - -o StrictHostKeyChecking=no \ - "$(printf "%s@%s" "$user" "$hostname")" \ - exit 2> /dev/null; do - [ "$attempts" -ge "$connection_attempts" ] && exit 1 - ((attempts++)) - printf "SSH connection attempt %d/%d failed. Retrying...\n" "$attempts" "$connection_attempts" - # sleep for Permission denied (publickey) errors - sleep 10 + -i ${private_key_file} \ + -o ConnectTimeout=10 \ + -o ForwardAgent=yes \ + -o IdentitiesOnly=yes \ + -o StrictHostKeyChecking=no \ + "$(printf "%s@%s" "$user" "$hostname")" \ + exit 2>/dev/null; do + [ "$attempts" -ge "$connection_attempts" ] && exit 1 + ((attempts++)) + printf "SSH connection attempt %d/%d failed. Retrying...\n" "$attempts" "$connection_attempts" + # sleep for Permission denied (publickey) errors + sleep 10 done diff --git a/evergreen/powercycle_exit.sh b/evergreen/powercycle_exit.sh index 44847473282..1d8ca977809 100644 --- a/evergreen/powercycle_exit.sh +++ b/evergreen/powercycle_exit.sh @@ -1,15 +1,15 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" # Test exits from here with specified exit_code. if [ -n "${exit_code}" ]; then - # Python program saved exit_code - exit_code=${exit_code} + # Python program saved exit_code + exit_code=${exit_code} elif [ -f error_exit.txt ]; then - # Bash trap exit_code - exit_code=$(cat error_exit.txt) + # Bash trap exit_code + exit_code=$(cat error_exit.txt) else - exit_code=0 + exit_code=0 fi echo "Exiting powercycle with code $exit_code" exit $exit_code diff --git a/evergreen/powercycle_run_test.sh b/evergreen/powercycle_run_test.sh index 4d82ea0ef7d..c959f6f473b 100644 --- a/evergreen/powercycle_run_test.sh +++ b/evergreen/powercycle_run_test.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -7,9 +7,9 @@ set -o errexit set -o verbose if [ "Windows_NT" = "$OS" ]; then - user=Administrator + user=Administrator else - user=$USER + user=$USER fi activate_venv @@ -17,7 +17,7 @@ activate_venv trap 'echo $? > error_exit.txt; exit 0' EXIT set +o errexit eval $python -u buildscripts/resmoke.py powercycle run \ - "--sshUserHost=$(printf "%s@%s" "$user" "${private_ip_address}") \ + "--sshUserHost=$(printf "%s@%s" "$user" "${private_ip_address}") \ --sshConnection=\"-i powercycle.pem\" \ --taskName=${task_name} \ ${run_powercycle_args}" diff --git a/evergreen/powercycle_save_artifacts.sh b/evergreen/powercycle_save_artifacts.sh index 615fab1c7fe..32eb04f6213 100644 --- a/evergreen/powercycle_save_artifacts.sh +++ b/evergreen/powercycle_save_artifacts.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -6,7 +6,7 @@ cd src set -o verbose if [ ! -f powercycle_ip_address.yml ]; then - exit 0 + exit 0 fi activate_venv diff --git a/evergreen/powercycle_sentinel_run.sh b/evergreen/powercycle_sentinel_run.sh index 861f2662239..f36f94dda1d 100755 --- a/evergreen/powercycle_sentinel_run.sh +++ b/evergreen/powercycle_sentinel_run.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src diff --git a/evergreen/powercycle_setup_host.sh b/evergreen/powercycle_setup_host.sh index 31c6042fb84..4bf0e9bdc8c 100644 --- a/evergreen/powercycle_setup_host.sh +++ b/evergreen/powercycle_setup_host.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src diff --git a/evergreen/powercycle_ssh_failure_exit.sh b/evergreen/powercycle_ssh_failure_exit.sh index fa0d824239e..2f11c7b62b3 100644 --- a/evergreen/powercycle_ssh_failure_exit.sh +++ b/evergreen/powercycle_ssh_failure_exit.sh @@ -1,10 +1,10 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src # Trigger a system failure if powercycle failed due to ssh access. if [ -n "${ec2_ssh_failure}" ]; then - echo "ec2_ssh_failure detected - $(cat powercycle_exit.yml)" - exit ${exit_code} + echo "ec2_ssh_failure detected - $(cat powercycle_exit.yml)" + exit ${exit_code} fi diff --git a/evergreen/powercycle_tasks_generate.sh b/evergreen/powercycle_tasks_generate.sh index ddfc54c4bf6..15b7555790e 100755 --- a/evergreen/powercycle_tasks_generate.sh +++ b/evergreen/powercycle_tasks_generate.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src diff --git a/evergreen/prelude.sh b/evergreen/prelude.sh index d30df9d9b96..7299ffb17d6 100755 --- a/evergreen/prelude.sh +++ b/evergreen/prelude.sh @@ -1,11 +1,11 @@ if [[ "$0" == *"/evergreen/prelude.sh" ]]; then - echo "ERROR: do not execute this script. source it instead. i.e.: . prelude.sh" - exit 1 + echo "ERROR: do not execute this script. source it instead. i.e.: . prelude.sh" + exit 1 fi set -o errexit # path the directory that contains this script. -evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" evergreen_workdir=$(realpath ${evergreen_dir}/../..) mkdir -p ${evergreen_workdir}/bin @@ -13,26 +13,26 @@ export PATH=$PATH:${evergreen_workdir}/bin unset evergreen_workdir function timeout_and_retry { - TIMEOUT=$1 - shift - RETRIES=3 + TIMEOUT=$1 + shift + RETRIES=3 - RET=0 - for i in $(seq $RETRIES); do - echo "Running command with ${TIMEOUT}s timeout: $@" - eval timeout $TIMEOUT $@ || RET=$? + RET=0 + for i in $(seq $RETRIES); do + echo "Running command with ${TIMEOUT}s timeout: $@" + eval timeout $TIMEOUT $@ || RET=$? - if [ $RET -eq 0 ]; then - break - elif [ $RET -eq 124 ]; then - echo "Command timed out after ${TIMEOUT}s on attempt $i: $@" - else - echo "Command failed with exitcode $RET after $i attempts: $@" - fi - sleep 1 - done + if [ $RET -eq 0 ]; then + break + elif [ $RET -eq 124 ]; then + echo "Command timed out after ${TIMEOUT}s on attempt $i: $@" + else + echo "Command failed with exitcode $RET after $i attempts: $@" + fi + sleep 1 + done - return $RET + return $RET } # let child shell processes use timeout_and_retry @@ -49,15 +49,15 @@ expansions_yaml="$evergreen_dir/../../expansions.yml" expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml" script="$evergreen_dir/../buildscripts/evergreen_expansions2bash.py" if [ "Windows_NT" = "$OS" ]; then - expansions_yaml=$(cygpath -w "$expansions_yaml") - expansions_default_yaml=$(cygpath -w "$expansions_default_yaml") - script=$(cygpath -w "$script") + expansions_yaml=$(cygpath -w "$expansions_yaml") + expansions_default_yaml=$(cygpath -w "$expansions_default_yaml") + script=$(cygpath -w "$script") fi -eval $(activate_venv > /dev/null && $python "$script" "$expansions_yaml" "$expansions_default_yaml") +eval $(activate_venv >/dev/null && $python "$script" "$expansions_yaml" "$expansions_default_yaml") if [ -n "$___expansions_error" ]; then - echo $___expansions_error - exit 1 + echo $___expansions_error + exit 1 fi unset expansions_yaml unset expansions_default_yaml @@ -65,29 +65,29 @@ unset script unset evergreen_dir function posix_workdir { - if [ "Windows_NT" = "$OS" ]; then - echo $(cygpath -u "${workdir}") - else - echo ${workdir} - fi + if [ "Windows_NT" = "$OS" ]; then + echo $(cygpath -u "${workdir}") + else + echo ${workdir} + fi } function set_sudo { - set -o > /tmp/settings.log - set +o errexit - grep errexit /tmp/settings.log | grep on - errexit_on=$? - # Set errexit "off". - set +o errexit - sudo= - # Use sudo, if it is supported. - sudo date > /dev/null 2>&1 - if [ $? -eq 0 ]; then - sudo=sudo - fi - # Set errexit "on", if previously enabled. - if [ $errexit_on -eq 0 ]; then - set -o errexit - fi + set -o >/tmp/settings.log + set +o errexit + grep errexit /tmp/settings.log | grep on + errexit_on=$? + # Set errexit "off". + set +o errexit + sudo= + # Use sudo, if it is supported. + sudo date >/dev/null 2>&1 + if [ $? -eq 0 ]; then + sudo=sudo + fi + # Set errexit "on", if previously enabled. + if [ $errexit_on -eq 0 ]; then + set -o errexit + fi } set +o errexit diff --git a/evergreen/prelude_db_contrib_tool.sh b/evergreen/prelude_db_contrib_tool.sh index ad92397cbd9..d47af6914dc 100755 --- a/evergreen/prelude_db_contrib_tool.sh +++ b/evergreen/prelude_db_contrib_tool.sh @@ -1,63 +1,63 @@ function setup_db_contrib_tool { - # check if db-contrib-tool is already installed - if [[ $(type -P db-contrib-tool) ]]; then - return 0 - fi + # check if db-contrib-tool is already installed + if [[ $(type -P db-contrib-tool) ]]; then + return 0 + fi - $python evergreen/download_db_contrib_tool.py + $python evergreen/download_db_contrib_tool.py } function use_db_contrib_tool_mongot { - # Checking that this is not a downstream patch on mongod created by mongot's patch trigger. - # In the case that it's not, download latest (eg HEAD of 10gen/mongot) or the - # release (eg currently running in production on Atlas) mongot binary. - arch=$(uname -i) - if [[ ! $(declare -p linux_x86_64_mongot_localdev_binary linux_aarch64_mongot_localdev_binary macos_x86_64_mongot_localdev_binary 2> /dev/null) ]]; then + # Checking that this is not a downstream patch on mongod created by mongot's patch trigger. + # In the case that it's not, download latest (eg HEAD of 10gen/mongot) or the + # release (eg currently running in production on Atlas) mongot binary. + arch=$(uname -i) + if [[ ! $(declare -p linux_x86_64_mongot_localdev_binary linux_aarch64_mongot_localdev_binary macos_x86_64_mongot_localdev_binary 2>/dev/null) ]]; then - if [ "${download_mongot_release}" = "true" ]; then - mongot_version="release" - else - mongot_version="latest" - fi + if [ "${download_mongot_release}" = "true" ]; then + mongot_version="release" + else + mongot_version="latest" + fi - if [[ "$OSTYPE" == "linux-gnu"* ]]; then - mongot_platform="linux" - elif [[ "$OSTYPE" == "darwin"* ]]; then - mongot_platform="macos" - else - echo "mongot is only supported on linux and mac and does not support ${OSTYPE}" - exit 1 - fi + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + mongot_platform="linux" + elif [[ "$OSTYPE" == "darwin"* ]]; then + mongot_platform="macos" + else + echo "mongot is only supported on linux and mac and does not support ${OSTYPE}" + exit 1 + fi - mongot_arch="x86_64" - # macos arm64 is not supported by mongot, but macos x86_64 runs on it successfully - if [[ $arch == "aarch64"* ]] && [[ "$OSTYPE" != "darwin"* ]]; then - mongot_arch="aarch64" - fi - echo "running: db-contrib-tool setup-mongot-repro-env ${mongot_version} --platform=${mongot_platform} --architecture=${mongot_arch} --installDir=." - # This should create the folder mongot-localdev, usually run at the root of mongo directory - db-contrib-tool setup-mongot-repro-env ${mongot_version} --platform=${mongot_platform} --architecture=${mongot_arch} --installDir=. - else - # This is a downstream patch, which means there is a patched mongot binary we need to install. - if [[ "$OSTYPE" == "linux-gnu"* ]]; then - if [[ $arch == "x86_64"* ]]; then - mongot_url=${linux_x86_64_mongot_localdev_binary} - elif [[ $arch == "aarch64"* ]]; then - mongot_url=${linux_aarch64_mongot_localdev_binary} - else - echo "mongot-localdev does not support ${arch}" - exit 1 - fi - elif [[ "$OSTYPE" == "darwin"* ]]; then - mongot_url=${macos_x86_64_mongot_localdev_binary} + mongot_arch="x86_64" + # macos arm64 is not supported by mongot, but macos x86_64 runs on it successfully + if [[ $arch == "aarch64"* ]] && [[ "$OSTYPE" != "darwin"* ]]; then + mongot_arch="aarch64" + fi + echo "running: db-contrib-tool setup-mongot-repro-env ${mongot_version} --platform=${mongot_platform} --architecture=${mongot_arch} --installDir=." + # This should create the folder mongot-localdev, usually run at the root of mongo directory + db-contrib-tool setup-mongot-repro-env ${mongot_version} --platform=${mongot_platform} --architecture=${mongot_arch} --installDir=. else - echo "mongot-localdev does not support ${OSTYPE}" - exit 1 + # This is a downstream patch, which means there is a patched mongot binary we need to install. + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + if [[ $arch == "x86_64"* ]]; then + mongot_url=${linux_x86_64_mongot_localdev_binary} + elif [[ $arch == "aarch64"* ]]; then + mongot_url=${linux_aarch64_mongot_localdev_binary} + else + echo "mongot-localdev does not support ${arch}" + exit 1 + fi + elif [[ "$OSTYPE" == "darwin"* ]]; then + mongot_url=${macos_x86_64_mongot_localdev_binary} + else + echo "mongot-localdev does not support ${OSTYPE}" + exit 1 + fi + echo "running curl ${mongot_url} | tar xvz" + # This should create the folder mongot-localdev, usually run at the root of mongo directory + curl ${mongot_url} | tar xvz fi - echo "running curl ${mongot_url} | tar xvz" - # This should create the folder mongot-localdev, usually run at the root of mongo directory - curl ${mongot_url} | tar xvz - fi - # Hack to remove BUILD.bazel file that can be lying around in mongot - rm -f ./mongot-localdev/bin/jdk/BUILD.bazel + # Hack to remove BUILD.bazel file that can be lying around in mongot + rm -f ./mongot-localdev/bin/jdk/BUILD.bazel } diff --git a/evergreen/prelude_mongo_task_generator.sh b/evergreen/prelude_mongo_task_generator.sh index dd6173446f0..0299925555c 100644 --- a/evergreen/prelude_mongo_task_generator.sh +++ b/evergreen/prelude_mongo_task_generator.sh @@ -1,8 +1,8 @@ function setup_mongo_task_generator { - if [ ! -f mongo-task-generator ]; then - curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v3.4.1/mongo-task-generator --output mongo-task-generator - chmod +x mongo-task-generator - fi + if [ ! -f mongo-task-generator ]; then + curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v3.4.1/mongo-task-generator --output mongo-task-generator + chmod +x mongo-task-generator + fi } ## Comment above and uncomment below to test unreleased mongo-task-generator changes that are diff --git a/evergreen/prelude_python.sh b/evergreen/prelude_python.sh index b3d2d7830cd..17dfc8b4979 100644 --- a/evergreen/prelude_python.sh +++ b/evergreen/prelude_python.sh @@ -1,21 +1,21 @@ if [ "Windows_NT" = "$OS" ]; then - python='/cygdrive/c/python/python310/python.exe' - echo "Executing on windows, setting python to ${python}" + python='/cygdrive/c/python/python310/python.exe' + echo "Executing on windows, setting python to ${python}" elif [ "$(uname)" = "Darwin" ]; then - python='/Library/Frameworks/Python.Framework/Versions/3.10/bin/python3' - echo "Executing on mac, setting python to ${python}" + python='/Library/Frameworks/Python.Framework/Versions/3.10/bin/python3' + echo "Executing on mac, setting python to ${python}" else - if [ -f /opt/mongodbtoolchain/v5/bin/python3 ]; then - python="/opt/mongodbtoolchain/v5/bin/python3" - echo "Found python in v5 toolchain, setting python to ${python}" - elif [ -f /opt/mongodbtoolchain/v4/bin/python3 ]; then - python="/opt/mongodbtoolchain/v4/bin/python3" - echo "Found python in v4 toolchain, setting python to ${python}" - elif [ -f "$(which python3)" ]; then - python=$(which python3) - echo "Could not find mongodbtoolchain python, using system python ${python}" - else - echo "Could not find python3." - return 1 - fi + if [ -f /opt/mongodbtoolchain/v5/bin/python3 ]; then + python="/opt/mongodbtoolchain/v5/bin/python3" + echo "Found python in v5 toolchain, setting python to ${python}" + elif [ -f /opt/mongodbtoolchain/v4/bin/python3 ]; then + python="/opt/mongodbtoolchain/v4/bin/python3" + echo "Found python in v4 toolchain, setting python to ${python}" + elif [ -f "$(which python3)" ]; then + python=$(which python3) + echo "Could not find mongodbtoolchain python, using system python ${python}" + else + echo "Could not find python3." + return 1 + fi fi diff --git a/evergreen/prelude_system_env_variables.sh b/evergreen/prelude_system_env_variables.sh index 4ad4ba073a2..c7fd26e2700 100755 --- a/evergreen/prelude_system_env_variables.sh +++ b/evergreen/prelude_system_env_variables.sh @@ -1,7 +1,7 @@ #!/bin/bash get_glibc_version() { - getconf GNU_LIBC_VERSION | cut -d ' ' -f 2 + getconf GNU_LIBC_VERSION | cut -d ' ' -f 2 } # Systems with glibc 2.34 or newer register custom rseq ABI @@ -11,15 +11,15 @@ get_glibc_version() { # use TCMalloc's per-CPU caches. We must ensure this environment variable is # set on problematic systems. configure_glibc_pthread_req() { - if [[ "$OSTYPE" == "linux-gnu"* ]]; then - local ver="$(get_glibc_version)" - local major="$(echo $ver | cut -d '.' -f 1)" - local minor="$(echo $ver | cut -d '.' -f 2)" - if ((major > 2 || ((major == 2 && minor >= 34)))); then - export GLIBC_TUNABLES="glibc.pthread.rseq=0" - echo "glibc version >= 2.34 detected, setting env variable GLIBC_TUNABLES=glibc.pthread.rseq=0" + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + local ver="$(get_glibc_version)" + local major="$(echo $ver | cut -d '.' -f 1)" + local minor="$(echo $ver | cut -d '.' -f 2)" + if ((major > 2 || ((major == 2 && minor >= 34)))); then + export GLIBC_TUNABLES="glibc.pthread.rseq=0" + echo "glibc version >= 2.34 detected, setting env variable GLIBC_TUNABLES=glibc.pthread.rseq=0" + fi fi - fi } configure_glibc_pthread_req diff --git a/evergreen/prelude_venv.sh b/evergreen/prelude_venv.sh index 4c87c194a98..d373cb3170c 100644 --- a/evergreen/prelude_venv.sh +++ b/evergreen/prelude_venv.sh @@ -1,52 +1,52 @@ function activate_venv { - # check if virtualenv is set up - if [ -d "${workdir}/venv" ]; then - # It's possible for activate to fail without stderr; as a result the cat operation (below) could fail. - # To mitigate this, create an empty error log. - # We're relying on the evergreen provided tmp directory here because Amazon Linux 2023 has an issue - # writing to /tmp/ on startup. - touch $TMPDIR/activate_error.log + # check if virtualenv is set up + if [ -d "${workdir}/venv" ]; then + # It's possible for activate to fail without stderr; as a result the cat operation (below) could fail. + # To mitigate this, create an empty error log. + # We're relying on the evergreen provided tmp directory here because Amazon Linux 2023 has an issue + # writing to /tmp/ on startup. + touch $TMPDIR/activate_error.log + if [ "Windows_NT" = "$OS" ]; then + # Need to quote the path on Windows to preserve the separator. + . "${workdir}/venv/Scripts/activate" 2>$TMPDIR/activate_error.log + else + . ${workdir}/venv/bin/activate 2>$TMPDIR/activate_error.log + fi + if [ $? -ne 0 ]; then + echo "Failed to activate virtualenv: $(cat $TMPDIR/activate_error.log)" + exit 1 + fi + python=python + else + if [ -z "$python" ]; then + echo "\$python is unset. This should never happen" + exit 1 + fi + python=${python} + echo "Could not find venv. Setting python to $python." + fi + if [ "Windows_NT" = "$OS" ]; then - # Need to quote the path on Windows to preserve the separator. - . "${workdir}/venv/Scripts/activate" 2> $TMPDIR/activate_error.log + export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)" + elif [ "$(uname)" = "Darwin" ]; then + #SERVER-75626 After activating the virtual environment under the mocos host. the PYTHONPATH setting + #is incorrect, and the site-packages directory of the virtual environment cannot be found in the sys.path. + python_version=$($python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') + export PYTHONPATH="${workdir}/venv/lib/python${python_version}/site-packages:${PYTHONPATH}:${workdir}/src" else - . ${workdir}/venv/bin/activate 2> $TMPDIR/activate_error.log - fi - if [ $? -ne 0 ]; then - echo "Failed to activate virtualenv: $(cat $TMPDIR/activate_error.log)" - exit 1 - fi - python=python - else - if [ -z "$python" ]; then - echo "\$python is unset. This should never happen" - exit 1 - fi - python=${python} - echo "Could not find venv. Setting python to $python." - fi + python_version=$($python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') + site_packages="${workdir}/venv/lib/python${python_version}/site-packages" + python -c "import sys; print(sys.path)" - if [ "Windows_NT" = "$OS" ]; then - export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)" - elif [ "$(uname)" = "Darwin" ]; then - #SERVER-75626 After activating the virtual environment under the mocos host. the PYTHONPATH setting - #is incorrect, and the site-packages directory of the virtual environment cannot be found in the sys.path. - python_version=$($python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') - export PYTHONPATH="${workdir}/venv/lib/python${python_version}/site-packages:${PYTHONPATH}:${workdir}/src" - else - python_version=$($python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') - site_packages="${workdir}/venv/lib/python${python_version}/site-packages" - python -c "import sys; print(sys.path)" - - # Check if site_packages is already in sys.path - in_sys_path=$($python -c "import sys; print('$site_packages' in sys.path)") - if [ "$in_sys_path" = "False" ]; then - export PYTHONPATH="${site_packages}:${PYTHONPATH}:${workdir}/src" - else - export PYTHONPATH="$PYTHONPATH:${workdir}/src" + # Check if site_packages is already in sys.path + in_sys_path=$($python -c "import sys; print('$site_packages' in sys.path)") + if [ "$in_sys_path" = "False" ]; then + export PYTHONPATH="${site_packages}:${PYTHONPATH}:${workdir}/src" + else + export PYTHONPATH="$PYTHONPATH:${workdir}/src" + fi + python -c "import sys; print(sys.path)" fi - python -c "import sys; print(sys.path)" - fi - echo "python set to $(which $python) and python version: $($python --version)" + echo "python set to $(which $python) and python version: $($python --version)" } diff --git a/evergreen/prelude_workdir.sh b/evergreen/prelude_workdir.sh index 2e39ba1f3b8..7a8f5f77a34 100644 --- a/evergreen/prelude_workdir.sh +++ b/evergreen/prelude_workdir.sh @@ -1,26 +1,26 @@ calculated_workdir=$(cd "$evergreen_dir/../.." && echo "$PWD") pwd_cygpath="$PWD" if [ "Windows_NT" = "$OS" ]; then - calculated_workdir=$(cygpath -w "$calculated_workdir") - pwd_cygpath=$(cygpath -w "$pwd_cygpath") + calculated_workdir=$(cygpath -w "$calculated_workdir") + pwd_cygpath=$(cygpath -w "$pwd_cygpath") fi if [ -z "$workdir" ]; then - workdir="$calculated_workdir" + workdir="$calculated_workdir" # skip this test on Windows. The directories will never match due to the many # different path types present on Windows+Cygwin elif [ "$workdir" != "$calculated_workdir" ] && [ "Windows_NT" != "$OS" ]; then - # if you move the checkout directory (ex: simple project config project), - # then this assertion will fail in the future. You need to update - # calculated_workdir, and all the relative directories in this file. - echo "\$workdir was specified, but didn't match \$calculated_workdir. Did the directory structure change? Update prelude.sh" - echo "\$workdir: $workdir" - echo "\$calculated_workdir: $calculated_workdir" - exit 1 + # if you move the checkout directory (ex: simple project config project), + # then this assertion will fail in the future. You need to update + # calculated_workdir, and all the relative directories in this file. + echo "\$workdir was specified, but didn't match \$calculated_workdir. Did the directory structure change? Update prelude.sh" + echo "\$workdir: $workdir" + echo "\$calculated_workdir: $calculated_workdir" + exit 1 fi if [ "$pwd_cygpath" != "$calculated_workdir" ]; then - echo "ERROR: Your script changed directory before loading prelude.sh. Don't do that" - echo "\$PWD: $PWD" - echo "\$calculated_workdir: $calculated_workdir" - exit 1 + echo "ERROR: Your script changed directory before loading prelude.sh. Don't do that" + echo "\$PWD: $PWD" + echo "\$calculated_workdir: $calculated_workdir" + exit 1 fi diff --git a/evergreen/query_tester_test_repo_setup.sh b/evergreen/query_tester_test_repo_setup.sh index ee9e7ae6dc4..63602507c6a 100644 --- a/evergreen/query_tester_test_repo_setup.sh +++ b/evergreen/query_tester_test_repo_setup.sh @@ -12,7 +12,7 @@ # - src/mongo/db/query/query_tester/tests//generated_tests/[test_dirs] that contains unpopulated test dirs for resmoke # to determine the roots. These test_dirs will get populated by the query_tester_sparse_checkout.sh script. -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" set -o errexit @@ -20,8 +20,8 @@ set -o verbose # Ensure that repo_name has been defined in evergreen tasks. if [ -z "${repo_name}" ]; then - echo "Error: No repository name provided." - exit 1 + echo "Error: No repository name provided." + exit 1 fi # tmp dir in which we run the git checkouts. @@ -36,18 +36,18 @@ mkdir -p "${repo_name}/generated_tests" commit_hash=$(grep "^${repo_name}:" test_repos.conf | awk -F ':' '{print $2}') if [ -z "$commit_hash" ]; then - echo "Error: Repository with name ${repo_name} not found in configuration file." - exit 1 + echo "Error: Repository with name ${repo_name} not found in configuration file." + exit 1 fi for i in {1..5}; do - git clone --sparse --filter=blob:none https://x-access-token:${github_token}@github.com/10gen/${repo_name}.git ${repo_name_local} && RET=0 && break || RET=$? && sleep 5 - echo "Failed to clone github.com:10gen/${repo_name}.git, retrying..." + git clone --sparse --filter=blob:none https://x-access-token:${github_token}@github.com/10gen/${repo_name}.git ${repo_name_local} && RET=0 && break || RET=$? && sleep 5 + echo "Failed to clone github.com:10gen/${repo_name}.git, retrying..." done if [ $RET -ne 0 ]; then - echo "Failed to clone git@github.com:10gen/${repo_name}.git" - exit $RET + echo "Failed to clone git@github.com:10gen/${repo_name}.git" + exit $RET fi cd ${repo_name_local} || exit 1 @@ -59,10 +59,10 @@ mkdir generated_tests/ # Make sure the commit hash refers to a valid commit in the repo. if [ $RESET_RET -ne 0 ]; then - echo "Failed to reset to commit ${commit_hash} in ${repo_name_local}." - exit $RESET_RET + echo "Failed to reset to commit ${commit_hash} in ${repo_name_local}." + exit $RESET_RET else - echo "Successfully reset ${repo_name_local} to commit ${commit_hash}." + echo "Successfully reset ${repo_name_local} to commit ${commit_hash}." fi # Create directories for all tests before checking them out. This diff --git a/evergreen/query_tester_test_sparse_checkout.sh b/evergreen/query_tester_test_sparse_checkout.sh index 20887c002d5..473f6a9e84f 100644 --- a/evergreen/query_tester_test_sparse_checkout.sh +++ b/evergreen/query_tester_test_sparse_checkout.sh @@ -6,7 +6,7 @@ # complete and execution can begin. # This test directory is then copied to the directory detected by resmoke's root selector. -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" set -o errexit @@ -14,8 +14,8 @@ set -o verbose # Ensure that repo_name has been defined in evergreen tasks. if [ -z "$1" ]; then - echo "Error: No repository name provided." - exit 1 + echo "Error: No repository name provided." + exit 1 fi repo_name="$1" @@ -28,19 +28,19 @@ cd src/src/mongo/db/query/query_tester/tests cd $repo_name_local ls ../${repo_name}/generated_tests | while read i; do - TEST_DIR=generated_tests/$i - echo "Checking out $TEST_DIR at $(date)" + TEST_DIR=generated_tests/$i + echo "Checking out $TEST_DIR at $(date)" - for i in {1..5}; do - git sparse-checkout add $TEST_DIR && RET=0 && break || RET=$? && sleep 5 - echo "git sparse-checkout add $TEST_DIR failed, retrying..." - done + for i in {1..5}; do + git sparse-checkout add $TEST_DIR && RET=0 && break || RET=$? && sleep 5 + echo "git sparse-checkout add $TEST_DIR failed, retrying..." + done - if [ $RET -ne 0 ]; then - echo "Failed to git sparse-checkout add $TEST_DIR" - exit $RET - fi + if [ $RET -ne 0 ]; then + echo "Failed to git sparse-checkout add $TEST_DIR" + exit $RET + fi - touch $TEST_DIR/.sparse-checkout-done - cp $TEST_DIR/* $TEST_DIR/.sparse-checkout-done ../$repo_name/$TEST_DIR/ + touch $TEST_DIR/.sparse-checkout-done + cp $TEST_DIR/* $TEST_DIR/.sparse-checkout-done ../$repo_name/$TEST_DIR/ done diff --git a/evergreen/randomized_multiversion_tasks_generate.sh b/evergreen/randomized_multiversion_tasks_generate.sh index f9e34b61bad..44eba5e8685 100644 --- a/evergreen/randomized_multiversion_tasks_generate.sh +++ b/evergreen/randomized_multiversion_tasks_generate.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src diff --git a/evergreen/record_mongodb_server_version.sh b/evergreen/record_mongodb_server_version.sh index 052d23d51b8..45a84c2f214 100755 --- a/evergreen/record_mongodb_server_version.sh +++ b/evergreen/record_mongodb_server_version.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -8,7 +8,7 @@ set -o verbose build_patch_id="${build_patch_id:-${reuse_compile_from}}" if [ -n "${build_patch_id}" ]; then - exit 0 + exit 0 fi -"$1" --version > "$2" +"$1" --version >"$2" diff --git a/evergreen/resmoke_tests_execute.sh b/evergreen/resmoke_tests_execute.sh index 68fb84834a1..1532b4b6064 100755 --- a/evergreen/resmoke_tests_execute.sh +++ b/evergreen/resmoke_tests_execute.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -12,194 +12,194 @@ set -o verbose if [[ ${disable_unit_tests} = "false" && ! -f ${skip_tests} ]]; then - # activate the virtualenv if it has been set up - activate_venv - # Install db-contrib-tool to symbolize crashes during resmoke suite runs - # This is not supported on Windows and MacOS, so doing it only on Linux - if [ "$(uname)" == "Linux" ]; then - setup_db_contrib_tool - fi - - if [[ -f "patch_test_tags.tgz" ]]; then - tags_build_variant="${build_variant}" - - if [[ "${build_variant}" =~ .*"-query-patch-only" ]]; then - # Use the RHEL 8 all feature flags variant for the classic engine variant. The original - # classic engine variant is not a required builder and therefore not captured in patch - # test failure history. - tags_build_variant="enterprise-rhel-8-64-bit-dynamic-all-feature-flags-required" + # activate the virtualenv if it has been set up + activate_venv + # Install db-contrib-tool to symbolize crashes during resmoke suite runs + # This is not supported on Windows and MacOS, so doing it only on Linux + if [ "$(uname)" == "Linux" ]; then + setup_db_contrib_tool fi - $python buildscripts/testmatrix/getdisplaytaskname.py "${task_name}" "${build_variant}" > display_task_name.txt - display_task_name=$(cat display_task_name.txt) + if [[ -f "patch_test_tags.tgz" ]]; then + tags_build_variant="${build_variant}" - tar -xzf patch_test_tags.tgz + if [[ "${build_variant}" =~ .*"-query-patch-only" ]]; then + # Use the RHEL 8 all feature flags variant for the classic engine variant. The original + # classic engine variant is not a required builder and therefore not captured in patch + # test failure history. + tags_build_variant="enterprise-rhel-8-64-bit-dynamic-all-feature-flags-required" + fi - calculated_tags_file_path="failedtesttags/${tags_build_variant}/${display_task_name}.yml" + $python buildscripts/testmatrix/getdisplaytaskname.py "${task_name}" "${build_variant}" >display_task_name.txt + display_task_name=$(cat display_task_name.txt) - if [[ -f $calculated_tags_file_path ]]; then - extra_args="$extra_args --tagFile=failedtesttags/${tags_build_variant}/${display_task_name}.yml --includeWithAnyTags=recent_failure,recent_change" - else - echo "calculated tags file does not exist: $calculated_tags_file_path" + tar -xzf patch_test_tags.tgz + + calculated_tags_file_path="failedtesttags/${tags_build_variant}/${display_task_name}.yml" + + if [[ -f $calculated_tags_file_path ]]; then + extra_args="$extra_args --tagFile=failedtesttags/${tags_build_variant}/${display_task_name}.yml --includeWithAnyTags=recent_failure,recent_change" + else + echo "calculated tags file does not exist: $calculated_tags_file_path" + fi fi - fi - # Set the TMPDIR environment variable to be a directory in the task's working - # directory so that temporary files created by processes spawned by resmoke.py get - # cleaned up after the task completes. This also ensures the spawned processes - # aren't impacted by limited space in the mount point for the /tmp directory. - export TMPDIR="${workdir}/tmp" - mkdir -p $TMPDIR + # Set the TMPDIR environment variable to be a directory in the task's working + # directory so that temporary files created by processes spawned by resmoke.py get + # cleaned up after the task completes. This also ensures the spawned processes + # aren't impacted by limited space in the mount point for the /tmp directory. + export TMPDIR="${workdir}/tmp" + mkdir -p $TMPDIR - if [ -f /proc/self/coredump_filter ]; then - # Set the shell process (and its children processes) to dump ELF headers (bit 4), - # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0). - echo 0x13 > /proc/self/coredump_filter + if [ -f /proc/self/coredump_filter ]; then + # Set the shell process (and its children processes) to dump ELF headers (bit 4), + # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0). + echo 0x13 >/proc/self/coredump_filter - if [ -f /sbin/sysctl ]; then - # Check that the core pattern is set explicitly on our distro image instead - # of being the OS's default value. This ensures that coredump names are consistent - # across distros and can be picked up by Evergreen. - core_pattern=$(/sbin/sysctl -n "kernel.core_pattern") - if [ "$core_pattern" = "dump_%e.%p.core" ]; then - echo "Enabling coredumps" - ulimit -c unlimited - fi + if [ -f /sbin/sysctl ]; then + # Check that the core pattern is set explicitly on our distro image instead + # of being the OS's default value. This ensures that coredump names are consistent + # across distros and can be picked up by Evergreen. + core_pattern=$(/sbin/sysctl -n "kernel.core_pattern") + if [ "$core_pattern" = "dump_%e.%p.core" ]; then + echo "Enabling coredumps" + ulimit -c unlimited + fi + fi fi - fi - if [ $(uname -s) == "Darwin" ]; then - core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile") - if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then - echo "Enabling coredumps" - ulimit -c unlimited + if [ $(uname -s) == "Darwin" ]; then + core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile") + if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then + echo "Enabling coredumps" + ulimit -c unlimited + fi fi - fi - extra_args="$extra_args --jobs=${resmoke_jobs}" + extra_args="$extra_args --jobs=${resmoke_jobs}" - if [ ${should_shuffle} = true ]; then - extra_args="$extra_args --shuffle" - elif [ ${should_shuffle} = false ]; then - extra_args="$extra_args --shuffleMode=off" - fi - - if [ ${continue_on_failure} = true ]; then - extra_args="$extra_args --continueOnFailure" - fi - - # We reduce the storage engine's cache size to reduce the likelihood of a mongod process - # being killed by the OOM killer. The --storageEngineCacheSizeGB command line option is only - # filled in with a default value here if one hasn't already been specified in the task's - # definition or build variant's definition. - set +o errexit - echo "${resmoke_args} ${test_flags}" | grep -q storageEngineCacheSizeGB - if [ $? -eq 1 ]; then - echo "${resmoke_args} ${test_flags}" | grep -q "\-\-storageEngine=inMemory" - if [ $? -eq 0 ]; then - # We use a default of 4GB for the InMemory storage engine. - extra_args="$extra_args --storageEngineCacheSizeGB=4" - else - # We use a default of 1GB for all other storage engines. - extra_args="$extra_args --storageEngineCacheSizeGB=1" + if [ ${should_shuffle} = true ]; then + extra_args="$extra_args --shuffle" + elif [ ${should_shuffle} = false ]; then + extra_args="$extra_args --shuffleMode=off" fi - fi - set -o errexit - # Reduce the JSHeapLimit for the serial_run task task on Code Coverage builder variant. - if [[ "${build_variant}" = "enterprise-rhel-8-64-bit-coverage" && "${task_name}" = "serial_run" ]]; then - extra_args="$extra_args --mongodSetParameter \"{'jsHeapLimitMB':10}\"" - fi + if [ ${continue_on_failure} = true ]; then + extra_args="$extra_args --continueOnFailure" + fi - # Introduce JS_GC_ZEAL to be used specifically under mongod/mongos. - if [[ "${build_variant}" = "enterprise-rhel-8-64-bit-dynamic-spider-monkey-dbg" && ! -z "${mongo_mozjs_options}" ]]; then - extra_args="$extra_args --mozjsJsGcZeal='${mongo_mozjs_options}'" - fi - - if [ "${is_patch}" = "true" ]; then - extra_args="$extra_args --patchBuild" - fi - - if [ "${skip_symbolization}" = "true" ]; then - extra_args="$extra_args --skipSymbolization" - fi - - # Add test selection flag based on patch parameter - if [ "${enable_evergreen_api_test_selection}" = "true" ]; then - extra_args="$extra_args --enableEvergreenApiTestSelection" - fi - - # Split comma separated list of strategies - IFS=',' read -a strategies <<< "$test_selection_strategies_array" - for strategy in "${strategies[@]}"; do - extra_args+=" --evergreenTestSelectionStrategy=${strategy}" - done - - path_value="$PATH:/data/multiversion" - - # Set the suite name to be the task name by default; unless overridden with the `suite` expansion. - suite_name=${task_name} - if [[ -n ${suite} ]]; then - suite_name=${suite} - fi - - resmoke_env_options="${gcov_environment} ${lang_environment} ${san_options} ${mozjs_options}" - echo $resmoke_env_options > resmoke_env_options.txt - - # The "resmoke_wrapper" expansion is used by the 'burn_in_tests' task to wrap the resmoke.py - # invocation. It doesn't set any environment variables and should therefore come last in - # this list of expansions. - set +o errexit - PATH="$path_value" \ - AWS_PROFILE=${aws_profile_remote} \ - eval \ - $resmoke_env_options \ - ${resmoke_wrapper} \ - $python buildscripts/resmoke.py run \ - ${resmoke_args} \ - $extra_args \ - ${test_flags} \ - --suites=${suite_name} \ - --log=${resmoke_logger} \ - --staggerJobs=on \ - --installDir=${install_dir} \ - --buildId=${build_id} \ - --distroId=${distro_id} \ - --executionNumber=${execution} \ - --projectName=${project} \ - --gitRevision=${revision} \ - --revisionOrderId=${revision_order_id} \ - --taskId=${task_id} \ - --taskName=${task_name} \ - --variantName=${build_variant} \ - --versionId=${version_id} \ - --requester=${requester} \ - --taskWorkDir='${workdir}' \ - --projectConfigPath ${evergreen_config_file_path} \ - --reportFile=report.json \ - --cedarReportFile=cedar_report.json - resmoke_exit_code=$? - set -o errexit - - # 74 is exit code for IOError on POSIX systems, which is raised when the machine is - # shutting down. - # - # 75 is exit code resmoke.py uses when the log output would be incomplete due to failing - # to communicate with logkeeper. - if [[ $resmoke_exit_code = 74 || $resmoke_exit_code = 75 ]]; then - echo $resmoke_exit_code > run_tests_infrastructure_failure - exit 0 - elif [ $resmoke_exit_code != 0 ]; then - # On failure save the resmoke exit code. - echo $resmoke_exit_code > resmoke_error_code - exit 0 - elif [ $resmoke_exit_code = 0 ]; then - # On success delete core files. + # We reduce the storage engine's cache size to reduce the likelihood of a mongod process + # being killed by the OOM killer. The --storageEngineCacheSizeGB command line option is only + # filled in with a default value here if one hasn't already been specified in the task's + # definition or build variant's definition. set +o errexit - core_files=$(/usr/bin/find -H .. \( -name "*.core" -o -name "*.mdmp" \) 2> /dev/null) - rm -rf $core_files + echo "${resmoke_args} ${test_flags}" | grep -q storageEngineCacheSizeGB + if [ $? -eq 1 ]; then + echo "${resmoke_args} ${test_flags}" | grep -q "\-\-storageEngine=inMemory" + if [ $? -eq 0 ]; then + # We use a default of 4GB for the InMemory storage engine. + extra_args="$extra_args --storageEngineCacheSizeGB=4" + else + # We use a default of 1GB for all other storage engines. + extra_args="$extra_args --storageEngineCacheSizeGB=1" + fi + fi set -o errexit - fi - exit $resmoke_exit_code + # Reduce the JSHeapLimit for the serial_run task task on Code Coverage builder variant. + if [[ "${build_variant}" = "enterprise-rhel-8-64-bit-coverage" && "${task_name}" = "serial_run" ]]; then + extra_args="$extra_args --mongodSetParameter \"{'jsHeapLimitMB':10}\"" + fi + + # Introduce JS_GC_ZEAL to be used specifically under mongod/mongos. + if [[ "${build_variant}" = "enterprise-rhel-8-64-bit-dynamic-spider-monkey-dbg" && ! -z "${mongo_mozjs_options}" ]]; then + extra_args="$extra_args --mozjsJsGcZeal='${mongo_mozjs_options}'" + fi + + if [ "${is_patch}" = "true" ]; then + extra_args="$extra_args --patchBuild" + fi + + if [ "${skip_symbolization}" = "true" ]; then + extra_args="$extra_args --skipSymbolization" + fi + + # Add test selection flag based on patch parameter + if [ "${enable_evergreen_api_test_selection}" = "true" ]; then + extra_args="$extra_args --enableEvergreenApiTestSelection" + fi + + # Split comma separated list of strategies + IFS=',' read -a strategies <<<"$test_selection_strategies_array" + for strategy in "${strategies[@]}"; do + extra_args+=" --evergreenTestSelectionStrategy=${strategy}" + done + + path_value="$PATH:/data/multiversion" + + # Set the suite name to be the task name by default; unless overridden with the `suite` expansion. + suite_name=${task_name} + if [[ -n ${suite} ]]; then + suite_name=${suite} + fi + + resmoke_env_options="${gcov_environment} ${lang_environment} ${san_options} ${mozjs_options}" + echo $resmoke_env_options >resmoke_env_options.txt + + # The "resmoke_wrapper" expansion is used by the 'burn_in_tests' task to wrap the resmoke.py + # invocation. It doesn't set any environment variables and should therefore come last in + # this list of expansions. + set +o errexit + PATH="$path_value" \ + AWS_PROFILE=${aws_profile_remote} \ + eval \ + $resmoke_env_options \ + ${resmoke_wrapper} \ + $python buildscripts/resmoke.py run \ + ${resmoke_args} \ + $extra_args \ + ${test_flags} \ + --suites=${suite_name} \ + --log=${resmoke_logger} \ + --staggerJobs=on \ + --installDir=${install_dir} \ + --buildId=${build_id} \ + --distroId=${distro_id} \ + --executionNumber=${execution} \ + --projectName=${project} \ + --gitRevision=${revision} \ + --revisionOrderId=${revision_order_id} \ + --taskId=${task_id} \ + --taskName=${task_name} \ + --variantName=${build_variant} \ + --versionId=${version_id} \ + --requester=${requester} \ + --taskWorkDir='${workdir}' \ + --projectConfigPath ${evergreen_config_file_path} \ + --reportFile=report.json \ + --cedarReportFile=cedar_report.json + resmoke_exit_code=$? + set -o errexit + + # 74 is exit code for IOError on POSIX systems, which is raised when the machine is + # shutting down. + # + # 75 is exit code resmoke.py uses when the log output would be incomplete due to failing + # to communicate with logkeeper. + if [[ $resmoke_exit_code = 74 || $resmoke_exit_code = 75 ]]; then + echo $resmoke_exit_code >run_tests_infrastructure_failure + exit 0 + elif [ $resmoke_exit_code != 0 ]; then + # On failure save the resmoke exit code. + echo $resmoke_exit_code >resmoke_error_code + exit 0 + elif [ $resmoke_exit_code = 0 ]; then + # On success delete core files. + set +o errexit + core_files=$(/usr/bin/find -H .. \( -name "*.core" -o -name "*.mdmp" \) 2>/dev/null) + rm -rf $core_files + set -o errexit + fi + + exit $resmoke_exit_code fi # end if [[ ${disable_unit_tests} && ! -f ${skip_tests|/dev/null} ]] diff --git a/evergreen/resmoke_tests_execute_bazel.sh b/evergreen/resmoke_tests_execute_bazel.sh index 1ebb8026200..0ddff3381f7 100644 --- a/evergreen/resmoke_tests_execute_bazel.sh +++ b/evergreen/resmoke_tests_execute_bazel.sh @@ -6,7 +6,7 @@ # Required environment variables: # * ${targets} - Resmoke bazel target, like //buildscripts/resmokeconfig:core -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -24,44 +24,44 @@ BAZEL_BINARY=$(bazel_get_binary_path) # for retries. TIMEOUT_CMD="" if [ -n "${build_timeout_seconds}" ]; then - TIMEOUT_CMD="timeout ${build_timeout_seconds}" + TIMEOUT_CMD="timeout ${build_timeout_seconds}" fi if [ ${should_shuffle} = true ]; then - bazel_args="${bazel_args} --test_arg=--shuffle" + bazel_args="${bazel_args} --test_arg=--shuffle" elif [ ${should_shuffle} = false ]; then - bazel_args="${bazel_args} --test_arg=--shuffleMode=off" + bazel_args="${bazel_args} --test_arg=--shuffleMode=off" fi if [ "${is_patch}" = "true" ]; then - bazel_args="${bazel_args} --test_arg=--patchBuild" + bazel_args="${bazel_args} --test_arg=--patchBuild" fi if [ "${skip_symbolization}" = "true" ]; then - bazel_args="${bazel_args} --test_arg=--skipSymbolization" + bazel_args="${bazel_args} --test_arg=--skipSymbolization" fi # Add test selection flag based on patch parameter if [ "${enable_evergreen_api_test_selection}" = "true" ]; then - bazel_args="${bazel_args} --test_arg=--enableEvergreenApiTestSelection" + bazel_args="${bazel_args} --test_arg=--enableEvergreenApiTestSelection" fi # Split comma separated list of strategies -IFS=',' read -a strategies <<< "$test_selection_strategies_array" +IFS=',' read -a strategies <<<"$test_selection_strategies_array" for strategy in "${strategies[@]}"; do - bazel_args+=" --test_arg=--evergreenTestSelectionStrategy=${strategy}" + bazel_args+=" --test_arg=--evergreenTestSelectionStrategy=${strategy}" done set +o errexit for i in {1..3}; do - eval ${TIMEOUT_CMD} ${BAZEL_BINARY} fetch ${bazel_args} ${targets} && RET=0 && break || RET=$? && sleep 60 - if [ $RET -eq 124 ]; then - echo "Bazel fetch timed out after ${build_timeout_seconds} seconds, retrying..." - else - echo "Bazel fetch failed, retrying..." - fi - $BAZEL_BINARY shutdown + eval ${TIMEOUT_CMD} ${BAZEL_BINARY} fetch ${bazel_args} ${targets} && RET=0 && break || RET=$? && sleep 60 + if [ $RET -eq 124 ]; then + echo "Bazel fetch timed out after ${build_timeout_seconds} seconds, retrying..." + else + echo "Bazel fetch failed, retrying..." + fi + $BAZEL_BINARY shutdown done eval ${BAZEL_BINARY} test ${bazel_args} ${targets} @@ -72,7 +72,7 @@ set -o errexit # For a target //path:test, the undeclared test outputs are in # bazel-testlogs/path/test/test.outputs/outputs.zip. Extract them # to the root of the task working directory -find bazel-testlogs/$(sed "s|//||;s|:|/|" <<< ${targets}) -name outputs.zip | xargs -I {} unzip -qo {} -d ../ +find bazel-testlogs/$(sed "s|//||;s|:|/|" <<<${targets}) -name outputs.zip | xargs -I {} unzip -qo {} -d ../ # Combine reports from potentially multiple tests/shards. find ../ -name report*.json | xargs $python buildscripts/combine_reports.py --no-report-exit -o report.json diff --git a/evergreen/resmoke_tests_runtime_validate.sh b/evergreen/resmoke_tests_runtime_validate.sh index aee2111ff92..47caf669ed6 100644 --- a/evergreen/resmoke_tests_runtime_validate.sh +++ b/evergreen/resmoke_tests_runtime_validate.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -8,7 +8,7 @@ set -o errexit activate_venv $python buildscripts/resmoke_tests_runtime_validate.py \ - --resmoke-report-file ./report.json \ - --project-id ${project_id} \ - --build-variant ${build_variant} \ - --task-name ${task_name} + --resmoke-report-file ./report.json \ + --project-id ${project_id} \ + --build-variant ${build_variant} \ + --task-name ${task_name} diff --git a/evergreen/restore_git_history_and_tags.sh b/evergreen/restore_git_history_and_tags.sh index 63200cb81ec..84acd847a5b 100644 --- a/evergreen/restore_git_history_and_tags.sh +++ b/evergreen/restore_git_history_and_tags.sh @@ -6,7 +6,7 @@ # Required environment variables: # * ${dir} - target directory -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" cd "$dir" @@ -17,22 +17,22 @@ retry_git="$DIR/retry_git.sh" # For older commits, evergreen will already unshallow the repository. if [ -f .git/shallow ]; then - echo "Restoring git history and tags" - # Older versions of git require this to be set, newer versions don't mind - git config extensions.partialClone origin + echo "Restoring git history and tags" + # Older versions of git require this to be set, newer versions don't mind + git config extensions.partialClone origin - # Git versions prior to 2.20.0 do not support --filter=tree:0, so we fall - # back to doing a blobless fetch instead. - required_version="2.20.0" - git_version=$(git --version | awk '{print $3}') - if [ "$(printf '%s\n' "$required_version" "$git_version" | sort -V | head -n1)" = "$required_version" ]; then - $retry_git fetch origin --filter=tree:0 --unshallow --tags - else - $retry_git fetch origin --filter=blob:none --unshallow --tags - fi + # Git versions prior to 2.20.0 do not support --filter=tree:0, so we fall + # back to doing a blobless fetch instead. + required_version="2.20.0" + git_version=$(git --version | awk '{print $3}') + if [ "$(printf '%s\n' "$required_version" "$git_version" | sort -V | head -n1)" = "$required_version" ]; then + $retry_git fetch origin --filter=tree:0 --unshallow --tags + else + $retry_git fetch origin --filter=blob:none --unshallow --tags + fi else - # Sometimes the tags necessary to describe a commit don't - # get fetched due to git version, so ensure they are. - echo "Ensuring git can describe the commit" - git describe 2> /dev/null || git fetch --tags + # Sometimes the tags necessary to describe a commit don't + # get fetched due to git version, so ensure they are. + echo "Ensuring git can describe the commit" + git describe 2>/dev/null || git fetch --tags fi diff --git a/evergreen/retry_git.sh b/evergreen/retry_git.sh index 3757cd2ef47..628de022508 100755 --- a/evergreen/retry_git.sh +++ b/evergreen/retry_git.sh @@ -14,17 +14,17 @@ RETRIES=5 DELAY=2 COUNT=0 while [ $COUNT -lt $RETRIES ]; do - "$REALGIT" "$@" - CODE=$? - if [ $CODE -eq 0 ]; then - RETRIES=0 - break - fi - ((COUNT = COUNT + 1)) - if [ $COUNT -lt $RETRIES ]; then - echo "Git command failed, retrying in $DELAY seconds..." >&2 - fi - sleep $DELAY + "$REALGIT" "$@" + CODE=$? + if [ $CODE -eq 0 ]; then + RETRIES=0 + break + fi + ((COUNT = COUNT + 1)) + if [ $COUNT -lt $RETRIES ]; then + echo "Git command failed, retrying in $DELAY seconds..." >&2 + fi + sleep $DELAY done exit "$CODE" diff --git a/evergreen/run_binary.sh b/evergreen/run_binary.sh index 329ae2e9391..bd8896137ed 100644 --- a/evergreen/run_binary.sh +++ b/evergreen/run_binary.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src diff --git a/evergreen/run_jstestfuzz/clone_repos.sh b/evergreen/run_jstestfuzz/clone_repos.sh index 8acbc78c36e..c2e41f87c53 100755 --- a/evergreen/run_jstestfuzz/clone_repos.sh +++ b/evergreen/run_jstestfuzz/clone_repos.sh @@ -4,13 +4,13 @@ set -o verbose cd src for i in {1..5}; do - git clone --depth 1 https://x-access-token:${github_token}@github.com/10gen/QA.git jstests/qa_tests && RET=0 && break || RET=$? && sleep 1 - echo "Failed to clone github.com:10gen/QA.git, retrying..." + git clone --depth 1 https://x-access-token:${github_token}@github.com/10gen/QA.git jstests/qa_tests && RET=0 && break || RET=$? && sleep 1 + echo "Failed to clone github.com:10gen/QA.git, retrying..." done if [ $RET -ne 0 ]; then - echo "Failed to clone github.com/10gen/QA.git" - exit $RET + echo "Failed to clone github.com/10gen/QA.git" + exit $RET fi cp -r src/mongo/db/modules/enterprise jstests/enterprise_tests diff --git a/evergreen/run_python_script.sh b/evergreen/run_python_script.sh index 746c229e9d8..dd393831a12 100644 --- a/evergreen/run_python_script.sh +++ b/evergreen/run_python_script.sh @@ -1,5 +1,5 @@ unset workdir -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" set -o errexit diff --git a/evergreen/run_python_script_with_report.sh b/evergreen/run_python_script_with_report.sh index 9a223e1d975..30959521c17 100644 --- a/evergreen/run_python_script_with_report.sh +++ b/evergreen/run_python_script_with_report.sh @@ -1,5 +1,5 @@ unset workdir -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" set -o verbose @@ -13,10 +13,10 @@ shift activate_venv echo $python $@ -echo $python $@ > python_report.log -$python "$@" &>> python_report.log +echo $python $@ >python_report.log +$python "$@" &>>python_report.log exit_code=$? -echo "Finished with exit code: $exit_code" >> python_report.log +echo "Finished with exit code: $exit_code" >>python_report.log $python ./buildscripts/simple_report.py --test-name "${report_test_name}" --log-file python_report.log --exit-code $exit_code exit $exit_code diff --git a/evergreen/run_upload_lock_push.sh b/evergreen/run_upload_lock_push.sh index 7b69d921e09..ee5f9875670 100755 --- a/evergreen/run_upload_lock_push.sh +++ b/evergreen/run_upload_lock_push.sh @@ -7,62 +7,62 @@ # source archives should be fine to be uploaded by whichever variant gets # there first declare -A ARTIFACTS=( - [${SERVER_TARBALL_PATH}]=${SERVER_TARBALL_KEY} - [${SOURCE_TARBALL_PATH}]=${SOURCE_TARBALL_KEY} - [${SERVER_TARBALL_SIGNATURE_PATH}]=${SERVER_TARBALL_SIGNATURE_KEY} - [${SOURCE_TARBALL_SIGNATURE_PATH}]=${SOURCE_TARBALL_SIGNATURE_KEY} - [${SERVER_TARBALL_SHA1_PATH}]=${SERVER_TARBALL_SHA1_KEY} - [${SOURCE_TARBALL_SHA1_PATH}]=${SOURCE_TARBALL_SHA1_KEY} - [${SERVER_TARBALL_SHA256_PATH}]=${SERVER_TARBALL_SHA256_KEY} - [${SOURCE_TARBALL_SHA256_PATH}]=${SOURCE_TARBALL_SHA256_KEY} - [${SERVER_TARBALL_MD5_PATH}]=${SERVER_TARBALL_MD5_KEY} - [${SOURCE_TARBALL_MD5_PATH}]=${SOURCE_TARBALL_MD5_KEY} + [${SERVER_TARBALL_PATH}]=${SERVER_TARBALL_KEY} + [${SOURCE_TARBALL_PATH}]=${SOURCE_TARBALL_KEY} + [${SERVER_TARBALL_SIGNATURE_PATH}]=${SERVER_TARBALL_SIGNATURE_KEY} + [${SOURCE_TARBALL_SIGNATURE_PATH}]=${SOURCE_TARBALL_SIGNATURE_KEY} + [${SERVER_TARBALL_SHA1_PATH}]=${SERVER_TARBALL_SHA1_KEY} + [${SOURCE_TARBALL_SHA1_PATH}]=${SOURCE_TARBALL_SHA1_KEY} + [${SERVER_TARBALL_SHA256_PATH}]=${SERVER_TARBALL_SHA256_KEY} + [${SOURCE_TARBALL_SHA256_PATH}]=${SOURCE_TARBALL_SHA256_KEY} + [${SERVER_TARBALL_MD5_PATH}]=${SERVER_TARBALL_MD5_KEY} + [${SOURCE_TARBALL_MD5_PATH}]=${SOURCE_TARBALL_MD5_KEY} ) # mongocryptd is only built for enterprise variants if [ -f "${CRYPTD_TARBALL_PATH}" ]; then - ARTIFACTS[${CRYPTD_TARBALL_PATH}]=${CRYPTD_TARBALL_KEY} - ARTIFACTS[${CRYPTD_TARBALL_SIGNATURE_PATH}]=${CRYPTD_TARBALL_SIGNATURE_KEY} - ARTIFACTS[${CRYPTD_TARBALL_SHA1_PATH}]=${CRYPTD_TARBALL_SHA1_KEY} - ARTIFACTS[${CRYPTD_TARBALL_SHA256_PATH}]=${CRYPTD_TARBALL_SHA256_KEY} - ARTIFACTS[${CRYPTD_TARBALL_MD5_PATH}]=${CRYPTD_TARBALL_MD5_KEY} + ARTIFACTS[${CRYPTD_TARBALL_PATH}]=${CRYPTD_TARBALL_KEY} + ARTIFACTS[${CRYPTD_TARBALL_SIGNATURE_PATH}]=${CRYPTD_TARBALL_SIGNATURE_KEY} + ARTIFACTS[${CRYPTD_TARBALL_SHA1_PATH}]=${CRYPTD_TARBALL_SHA1_KEY} + ARTIFACTS[${CRYPTD_TARBALL_SHA256_PATH}]=${CRYPTD_TARBALL_SHA256_KEY} + ARTIFACTS[${CRYPTD_TARBALL_MD5_PATH}]=${CRYPTD_TARBALL_MD5_KEY} fi # mongohouse only built sometimes # we do not sign mongohouse, so no detached signature and no checksums if [ -f "${MONGOHOUSE_TARBALL_PATH}" ]; then - ARTIFACTS[${MONGOHOUSE_TARBALL_PATH}]=${MONGOHOUSE_TARBALL_KEY} + ARTIFACTS[${MONGOHOUSE_TARBALL_PATH}]=${MONGOHOUSE_TARBALL_KEY} fi # debug symbols are only built sometimes # not clear which variants that is the case for if [ -f "${DEBUG_SYMBOLS_TARBALL_PATH}" ]; then - ARTIFACTS[${DEBUG_SYMBOLS_TARBALL_PATH}]=${DEBUG_SYMBOLS_TARBALL_KEY} - ARTIFACTS[${DEBUG_SYMBOLS_TARBALL_SIGNATURE_PATH}]=${DEBUG_SYMBOLS_TARBALL_SIGNATURE_KEY} - ARTIFACTS[${DEBUG_SYMBOLS_TARBALL_SHA1_PATH}]=${DEBUG_SYMBOLS_TARBALL_SHA1_KEY} - ARTIFACTS[${DEBUG_SYMBOLS_TARBALL_SHA256_PATH}]=${DEBUG_SYMBOLS_TARBALL_SHA256_KEY} - ARTIFACTS[${DEBUG_SYMBOLS_TARBALL_MD5_PATH}]=${DEBUG_SYMBOLS_TARBALL_MD5_KEY} + ARTIFACTS[${DEBUG_SYMBOLS_TARBALL_PATH}]=${DEBUG_SYMBOLS_TARBALL_KEY} + ARTIFACTS[${DEBUG_SYMBOLS_TARBALL_SIGNATURE_PATH}]=${DEBUG_SYMBOLS_TARBALL_SIGNATURE_KEY} + ARTIFACTS[${DEBUG_SYMBOLS_TARBALL_SHA1_PATH}]=${DEBUG_SYMBOLS_TARBALL_SHA1_KEY} + ARTIFACTS[${DEBUG_SYMBOLS_TARBALL_SHA256_PATH}]=${DEBUG_SYMBOLS_TARBALL_SHA256_KEY} + ARTIFACTS[${DEBUG_SYMBOLS_TARBALL_MD5_PATH}]=${DEBUG_SYMBOLS_TARBALL_MD5_KEY} fi # MSIs are only built on windows # note there is no detached signature file if [ -f "${MSI_PATH}" ]; then - ARTIFACTS[${MSI_PATH}]=${MSI_KEY} - ARTIFACTS[${MSI_SHA1_PATH}]=${MSI_SHA1_KEY} - ARTIFACTS[${MSI_SHA256_PATH}]=${MSI_SHA256_KEY} - ARTIFACTS[${MSI_MD5_PATH}]=${MSI_MD5_KEY} + ARTIFACTS[${MSI_PATH}]=${MSI_KEY} + ARTIFACTS[${MSI_SHA1_PATH}]=${MSI_SHA1_KEY} + ARTIFACTS[${MSI_SHA256_PATH}]=${MSI_SHA256_KEY} + ARTIFACTS[${MSI_MD5_PATH}]=${MSI_MD5_KEY} fi set -o verbose for path in "${!ARTIFACTS[@]}"; do - key=${ARTIFACTS[${path}]} - podman run \ - -v $(pwd):$(pwd) \ - -w $(pwd) \ - --env-host \ - ${UPLOAD_LOCK_IMAGE} \ - -key=${key} -tag=task-id=${EVERGREEN_TASK_ID} ${path} + key=${ARTIFACTS[${path}]} + podman run \ + -v $(pwd):$(pwd) \ + -w $(pwd) \ + --env-host \ + ${UPLOAD_LOCK_IMAGE} \ + -key=${key} -tag=task-id=${EVERGREEN_TASK_ID} ${path} done diff --git a/evergreen/sasl_windows_cyrussasl_setup.sh b/evergreen/sasl_windows_cyrussasl_setup.sh index 149bb563325..673c80f6c37 100644 --- a/evergreen/sasl_windows_cyrussasl_setup.sh +++ b/evergreen/sasl_windows_cyrussasl_setup.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" readonly k_cyrussasl_plugin_filename="cyrus_sasl_windows_test_plugin.dll" @@ -7,8 +7,8 @@ readonly k_cyrussasl_plugin_dir="/cygdrive/c/CMU/bin/sasl2" plugin_path="$(find . -name "*${k_cyrussasl_plugin_filename}")" if [[ -z "$plugin_path" ]]; then - echo >&2 "Could not find ${k_cyrussasl_plugin_filename} from path '$(pwd)' !" - exit 1 + echo >&2 "Could not find ${k_cyrussasl_plugin_filename} from path '$(pwd)' !" + exit 1 fi echo "Configuring CyrusSASL plugin .dll from '$plugin_path'" diff --git a/evergreen/sasl_windows_cyrussasl_teardown.sh b/evergreen/sasl_windows_cyrussasl_teardown.sh index 60e65055643..5c87d5d820d 100644 --- a/evergreen/sasl_windows_cyrussasl_teardown.sh +++ b/evergreen/sasl_windows_cyrussasl_teardown.sh @@ -1,8 +1,8 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" if [ "${task_name}" != "sasl_windows_cyrussasl" ]; then - exit 0 + exit 0 fi echo "Cleaning up Windows CyrusSASL Test Artifacts" @@ -10,8 +10,8 @@ echo "Cleaning up Windows CyrusSASL Test Artifacts" readonly k_cyrussasl_default_dir_root="/cygdrive/c/CMU" if [[ ! -d "$k_cyrussasl_default_dir_root" ]]; then - echo "Could not find $k_cyrussasl_default_dir_root to cleanup..." - exit 0 + echo "Could not find $k_cyrussasl_default_dir_root to cleanup..." + exit 0 fi rm -rf "$k_cyrussasl_default_dir_root" diff --git a/evergreen/selinux_run_test.sh b/evergreen/selinux_run_test.sh index c7ec9502a46..00af33774b3 100755 --- a/evergreen/selinux_run_test.sh +++ b/evergreen/selinux_run_test.sh @@ -14,14 +14,14 @@ set -o errexit -readonly k_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +readonly k_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" if [ "$BYPASS_PRELUDE" != "yes" ]; then - . "$k_dir/prelude.sh" - activate_venv - readonly k_src="src" + . "$k_dir/prelude.sh" + activate_venv + readonly k_src="src" else - readonly k_src="$SRC" + readonly k_src="$SRC" fi # If no selinux hostname is defined by external env, then we are running through evergreen, which has dumped spawn host @@ -31,10 +31,10 @@ fi # Also note that $workdir here is a built-in expansion from evergreen: see more info at # https://github.com/evergreen-ci/evergreen/blob/main/docs/Project-Configuration/Project-Configuration-Files.md#default-expansions if [ "$SELINUX_HOSTNAME" == "" ]; then - readonly k_selinux_hostname="$(tr -d '"[]{}' < "$workdir"/$k_src/hosts.yml | cut -d , -f 1 | awk -F : '{print $2}')" - cat "$workdir"/$k_src/hosts.yml + readonly k_selinux_hostname="$(tr -d '"[]{}' <"$workdir"/$k_src/hosts.yml | cut -d , -f 1 | awk -F : '{print $2}')" + cat "$workdir"/$k_src/hosts.yml else - readonly k_selinux_hostname="$SELINUX_HOSTNAME" + readonly k_selinux_hostname="$SELINUX_HOSTNAME" fi # SELINUX_USER injected from evergreen config, do not change @@ -43,76 +43,76 @@ readonly k_host="${SELINUX_USER}@${k_selinux_hostname}" # Obtain the ssh key and properties from expansions.yml, output from evergreen via the expansions.write command # (https://github.com/evergreen-ci/evergreen/blob/main/docs/Project-Configuration/Project-Commands.md#expansionswrite) if [ "$SSH_KEY" == "" ]; then - readonly k_ssh_key="$workdir/selinux.pem" + readonly k_ssh_key="$workdir/selinux.pem" - "$workdir"/$k_src/buildscripts/yaml_key_value.py --yamlFile="$workdir"/expansions.yml \ - --yamlKey=__project_aws_ssh_key_value > "$k_ssh_key" + "$workdir"/$k_src/buildscripts/yaml_key_value.py --yamlFile="$workdir"/expansions.yml \ + --yamlKey=__project_aws_ssh_key_value >"$k_ssh_key" - chmod 600 "$k_ssh_key" + chmod 600 "$k_ssh_key" - result="$(openssl rsa -in "$k_ssh_key" -check -noout | tee /dev/stderr)" + result="$(openssl rsa -in "$k_ssh_key" -check -noout | tee /dev/stderr)" - if [ "$result" != "RSA key ok" ]; then - exit 1 - fi + if [ "$result" != "RSA key ok" ]; then + exit 1 + fi else - readonly k_ssh_key="$SSH_KEY" + readonly k_ssh_key="$SSH_KEY" fi readonly k_ssh_options="-i $k_ssh_key -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" function copy_sources_to_target() { - rsync -ar -e "ssh $k_ssh_options" \ - --exclude 'tmp' --exclude 'build' --exclude '.*' \ - "$workdir"/$k_src/* "$k_host": + rsync -ar -e "ssh $k_ssh_options" \ + --exclude 'tmp' --exclude 'build' --exclude '.*' \ + "$workdir"/$k_src/* "$k_host": - return $? + return $? } function configure_target_machine() { - ssh $k_ssh_options "$k_host" evergreen/selinux_test_setup.sh - return $? + ssh $k_ssh_options "$k_host" evergreen/selinux_test_setup.sh + return $? } function execute_tests_on_target() { - ssh $k_ssh_options "$k_host" evergreen/selinux_test_executor.sh "$1" - return $? + ssh $k_ssh_options "$k_host" evergreen/selinux_test_executor.sh "$1" + return $? } function check_remote_connectivity() { - ssh -q $k_ssh_options -o ConnectTimeout=10 "$k_host" echo "I am working" - return $? + ssh -q $k_ssh_options -o ConnectTimeout=10 "$k_host" echo "I am working" + return $? } function retry_command() { - local connection_attempts=$1 - local cmd="$2" - shift 2 #eat the first 2 parameters to pass on any remaining to the calling function + local connection_attempts=$1 + local cmd="$2" + shift 2 #eat the first 2 parameters to pass on any remaining to the calling function - local attempts=0 - set +o errexit + local attempts=0 + set +o errexit - while true; do - "$cmd" "$@" + while true; do + "$cmd" "$@" - local result=$? + local result=$? - if [[ $result -eq 0 ]]; then - set -o errexit - return $result - fi + if [[ $result -eq 0 ]]; then + set -o errexit + return $result + fi - if [[ $attempts -ge $connection_attempts ]]; then - printf "%s failed after %d attempts with final error code %s.\n" "$cmd" "$attempts" "$result" - exit 1 - fi + if [[ $attempts -ge $connection_attempts ]]; then + printf "%s failed after %d attempts with final error code %s.\n" "$cmd" "$attempts" "$result" + exit 1 + fi - sleep 10 - ((attempts++)) + sleep 10 + ((attempts++)) - done + done } echo "===> Checking for remote connectivity..." @@ -126,16 +126,16 @@ retry_command 5 configure_target_machine echo "===> Executing tests..." readonly list="$( - cd src + cd src - # $TEST_LIST defined in evegreen "run selinux tests" function, do not change - for x in $TEST_LIST; do echo "$x"; done + # $TEST_LIST defined in evegreen "run selinux tests" function, do not change + for x in $TEST_LIST; do echo "$x"; done )" for test in $list; do - execute_tests_on_target "$test" - res="$?" - if [[ $res -ne 0 ]]; then - exit "$res" - fi + execute_tests_on_target "$test" + res="$?" + if [[ $res -ne 0 ]]; then + exit "$res" + fi done diff --git a/evergreen/selinux_test_executor.sh b/evergreen/selinux_test_executor.sh index d384e231bba..217a59c4be5 100755 --- a/evergreen/selinux_test_executor.sh +++ b/evergreen/selinux_test_executor.sh @@ -9,44 +9,44 @@ return_code=1 export PATH="$(dirname "$k_mongo"):$PATH" function print_err() { - echo "$@" >&2 + echo "$@" >&2 } function monitor_log() { - sed "s!^!mongod| $(date '+%F %H-%M-%S') !" <(sudo --non-interactive tail -f $k_log_path) + sed "s!^!mongod| $(date '+%F %H-%M-%S') !" <(sudo --non-interactive tail -f $k_log_path) } function output_ausearch() { - local cmd_parameters="AVC,USER_AVC,SELINUX_ERR,USER_SELINUX_ERR" + local cmd_parameters="AVC,USER_AVC,SELINUX_ERR,USER_SELINUX_ERR" - echo "" - echo "====== SELinux errors (ausearch -m $cmd_parameters): ======" - sudo --non-interactive ausearch -m $cmd_parameters -ts $1 + echo "" + echo "====== SELinux errors (ausearch -m $cmd_parameters): ======" + sudo --non-interactive ausearch -m $cmd_parameters -ts $1 } function output_journalctl() { - echo "" - echo "============================== journalctl =========================================" - sudo --non-interactive journalctl --no-pager --catalog --since="$1" | grep -i mongo + echo "" + echo "============================== journalctl =========================================" + sudo --non-interactive journalctl --no-pager --catalog --since="$1" | grep -i mongo } function fail_and_exit_err() { - echo "" - echo "===================================================================================" - echo "++++++++ Test failed, outputting last 5 seconds of additional log info ++++++++++++" - echo "===================================================================================" - output_ausearch "$(date --utc --date='5 seconds ago' '+%x %H:%M:%S')" - output_journalctl "$(date --utc --date='5 seconds ago' +'%Y-%m-%d %H:%M:%S')" + echo "" + echo "===================================================================================" + echo "++++++++ Test failed, outputting last 5 seconds of additional log info ++++++++++++" + echo "===================================================================================" + output_ausearch "$(date --utc --date='5 seconds ago' '+%x %H:%M:%S')" + output_journalctl "$(date --utc --date='5 seconds ago' +'%Y-%m-%d %H:%M:%S')" - echo "" - echo "==== FAIL: $1 ====" - exit 1 + echo "" + echo "==== FAIL: $1 ====" + exit 1 } function create_mongo_config() { - echo "Writing /etc/mongod.conf for $k_test_path:" - "$k_mongo" --nodb --norc --quiet --eval=' + echo "Writing /etc/mongod.conf for $k_test_path:" + "$k_mongo" --nodb --norc --quiet --eval=' const {TestDefinition} = await import("'"$k_test_path"'"); assert(TestDefinition); const test = new TestDefinition(); @@ -56,44 +56,44 @@ function create_mongo_config() { } function start_mongod() { - # Start mongod and if it won't come up, fail and exit + # Start mongod and if it won't come up, fail and exit - sudo --non-interactive systemctl start mongod \ - && sudo --non-interactive systemctl status mongod || ( - fail_and_exit_err "systemd failed to start mongod server!" - ) + sudo --non-interactive systemctl start mongod && + sudo --non-interactive systemctl status mongod || ( + fail_and_exit_err "systemd failed to start mongod server!" + ) } function wait_for_mongod_to_accept_connections() { - # Once the mongod process starts via systemd, it can still take a couple of seconds - # to set up and accept connections... we will wait for log id 23016 to show up - # indicating that the server is ready to accept incoming connections before starting the tests + # Once the mongod process starts via systemd, it can still take a couple of seconds + # to set up and accept connections... we will wait for log id 23016 to show up + # indicating that the server is ready to accept incoming connections before starting the tests - local server_ready=0 - local wait_seconds=2 - local wait_retries_max=30 - local wait_retries=0 + local server_ready=0 + local wait_seconds=2 + local wait_retries_max=30 + local wait_retries=0 - while [[ $wait_retries -le $wait_retries_max ]]; do - local server_status="$(grep 23016 $k_log_path || echo "")" + while [[ $wait_retries -le $wait_retries_max ]]; do + local server_status="$(grep 23016 $k_log_path || echo "")" - if [ "$server_status" != "" ]; then - server_ready=1 - break + if [ "$server_status" != "" ]; then + server_ready=1 + break + fi + + sleep $wait_seconds + ((wait_retries++)) + done + + if [ ! $server_ready ]; then + fail_and_exit_err "failed to connect to mongod server after waiting for $(($wait_seconds * $wait_retries)) seconds!" fi - - sleep $wait_seconds - ((wait_retries++)) - done - - if [ ! $server_ready ]; then - fail_and_exit_err "failed to connect to mongod server after waiting for $(($wait_seconds * $wait_retries)) seconds!" - fi } function clear_mongo_config() { - # stop mongod, zero mongo log, clean up database, set all booleans to off - sudo --non-interactive bash -c ' + # stop mongod, zero mongo log, clean up database, set all booleans to off + sudo --non-interactive bash -c ' systemctl stop mongod rm -f '"$k_log_path"' @@ -110,11 +110,11 @@ function clear_mongo_config() { } function exit_with_code() { - exit $return_code + exit $return_code } function setup_test_definition() { - "$k_mongo" --nodb --norc --quiet --eval=' + "$k_mongo" --nodb --norc --quiet --eval=' const {TestDefinition} = await import("'"$k_test_path"'"); assert(TestDefinition); (() => { @@ -126,7 +126,7 @@ function setup_test_definition() { } function run_test() { - "$k_mongo" --norc --gssapiServiceName=mockservice --eval=' + "$k_mongo" --norc --gssapiServiceName=mockservice --eval=' const {TestDefinition} = await import("'"$k_test_path"'"); assert(TestDefinition); @@ -141,24 +141,24 @@ function run_test() { } ' || fail_and_exit_err "Test failed" - echo "SUCCESS: $k_test_path" + echo "SUCCESS: $k_test_path" } if [ ! -f "$k_mongo" ]; then - print_err "Mongo shell at $k_mongo is missing" - exit 1 + print_err "Mongo shell at $k_mongo is missing" + exit 1 fi if [ ! -f "$k_test_path" ]; then - print_err "No test supplied or test file not found. Run:" - print_err "$(basename "${BASH_SOURCE[0]}") " - exit 1 + print_err "No test supplied or test file not found. Run:" + print_err "$(basename "${BASH_SOURCE[0]}") " + exit 1 fi # Ensure file containing tests is valid before executing if ! "$k_mongo" --nodb --norc --quiet "$k_test_path"; then - print_err "File $k_test_path has syntax errors" - exit 1 + print_err "File $k_test_path has syntax errors" + exit 1 fi echo "STARTING TEST: $k_test_path" diff --git a/evergreen/selinux_test_setup.sh b/evergreen/selinux_test_setup.sh index 83356ceb7fe..331d3b7030a 100755 --- a/evergreen/selinux_test_setup.sh +++ b/evergreen/selinux_test_setup.sh @@ -7,44 +7,44 @@ set -o xtrace set -o errexit function apply_selinux_policy() { - echo "==== Applying SELinux policy now" - rm -rf mongodb-selinux - git clone https://github.com/mongodb/mongodb-selinux - cd mongodb-selinux - make - sudo make install + echo "==== Applying SELinux policy now" + rm -rf mongodb-selinux + git clone https://github.com/mongodb/mongodb-selinux + cd mongodb-selinux + make + sudo make install } # on evergreen images /tmp is usually linked to /data/tmp, which interferes # with selinux, as it does not recognize it as tmp_t domain if [ -L /tmp ]; then - sudo --non-interactive rm /tmp - sudo --non-interactive mkdir /tmp - sudo --non-interactive systemctl start tmp.mount + sudo --non-interactive rm /tmp + sudo --non-interactive mkdir /tmp + sudo --non-interactive systemctl start tmp.mount fi # selinux policy should work both when applied before and after install # we will randomly apply it before or after installation is completed SEORDER="$(($RANDOM % 2))" if [ "$SEORDER" == "0" ]; then - apply_selinux_policy + apply_selinux_policy fi pkg="$(find "$HOME"/repo -name 'mongodb-*-server-*.x86_64.rpm' | tee /dev/stderr)" if ! sudo --non-interactive rpm --install --verbose --verbose --hash --nodeps "$pkg"; then - if [ "$?" -gt "1" ]; then exit 1; fi # exit code 1 is OK + if [ "$?" -gt "1" ]; then exit 1; fi # exit code 1 is OK fi if [ "$SEORDER" == "1" ]; then - apply_selinux_policy + apply_selinux_policy fi # install packages needed by check_has_tag.py PYTHON=/opt/mongodbtoolchain/v5/bin/python3 if [[ (-f "$PYTHON" || -L "$PYTHON") && -x "$PYTHON" ]]; then - echo "==== Found python3 in $PYTHON" - $PYTHON -m pip install pyyaml + echo "==== Found python3 in $PYTHON" + $PYTHON -m pip install pyyaml else - echo "==== Could not find $PYTHON; needed by SELinux tests" - exit 1 + echo "==== Could not find $PYTHON; needed by SELinux tests" + exit 1 fi diff --git a/evergreen/stitch_support_run_tests.sh b/evergreen/stitch_support_run_tests.sh index 63390093e44..40b547b79bb 100755 --- a/evergreen/stitch_support_run_tests.sh +++ b/evergreen/stitch_support_run_tests.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src diff --git a/evergreen/todos_check.sh b/evergreen/todos_check.sh index 42c20a20547..6ee743fa25e 100644 --- a/evergreen/todos_check.sh +++ b/evergreen/todos_check.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src @@ -12,14 +12,14 @@ set -o errexit # properly deal with any special characters that could cause issues (like "). To # do this, we will write it out to a file, then read that file into a variable. if [ "${is_commit_queue}" == "true" ]; then - cat > commit_message.txt << END_OF_COMMIT_MSG + cat >commit_message.txt < ./body.json +echo "$body" >./body.json echo ".................." echo "custom build endpoint body" @@ -54,26 +54,26 @@ cat ./body.json echo "uploading custom build" response=$(curl -sS --fail-with-body \ - -X "POST" \ - --digest \ - --header "Content-Type: application/json" \ - --data @body.json \ - -u "${CLOUD_DEV_API_PUBLIC_KEY}:${CLOUD_DEV_API_PRIVATE_KEY}" \ - https://cloud-dev.mongodb.com/api/private/nds/customMongoDbBuild) + -X "POST" \ + --digest \ + --header "Content-Type: application/json" \ + --data @body.json \ + -u "${CLOUD_DEV_API_PUBLIC_KEY}:${CLOUD_DEV_API_PRIVATE_KEY}" \ + https://cloud-dev.mongodb.com/api/private/nds/customMongoDbBuild) result=$? echo $response | jq if [ $result -eq 0 ]; then - echo "successful custom build upload" - exit 0 -else - if [[ $response == *"DUPLICATE_MONGODB_BUILD_NAME"* ]]; then - echo "trueName already exists, skipping upload" + echo "successful custom build upload" exit 0 - fi +else + if [[ $response == *"DUPLICATE_MONGODB_BUILD_NAME"* ]]; then + echo "trueName already exists, skipping upload" + exit 0 + fi - echo "failed to upload" - exit 1 + echo "failed to upload" + exit 1 fi diff --git a/evergreen/verify_build_output_present.sh b/evergreen/verify_build_output_present.sh index 21b553e5237..9e6552f7d2d 100644 --- a/evergreen/verify_build_output_present.sh +++ b/evergreen/verify_build_output_present.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" cd src diff --git a/evergreen/wait_for_resmoke_to_shutdown.sh b/evergreen/wait_for_resmoke_to_shutdown.sh index 32b463b11f1..3dd09ee040c 100755 --- a/evergreen/wait_for_resmoke_to_shutdown.sh +++ b/evergreen/wait_for_resmoke_to_shutdown.sh @@ -1,22 +1,22 @@ proc="resmoke.py" if [ "Windows_NT" = "$OS" ]; then - check_resmoke() { - resmoke_info=$(wmic process | grep resmoke.py) - } - while [ 1 ]; do - check_resmoke - if ! [[ "$resmoke_info" =~ .*"$proc".* ]]; then - break - fi - sleep 5 - done + check_resmoke() { + resmoke_info=$(wmic process | grep resmoke.py) + } + while [ 1 ]; do + check_resmoke + if ! [[ "$resmoke_info" =~ .*"$proc".* ]]; then + break + fi + sleep 5 + done else - get_pids() { proc_pids=$(pgrep -f $1); } - while [ 1 ]; do - get_pids $proc - if [ -z "$proc_pids" ]; then - break - fi - sleep 5 - done + get_pids() { proc_pids=$(pgrep -f $1); } + while [ 1 ]; do + get_pids $proc + if [ -z "$proc_pids" ]; then + break + fi + sleep 5 + done fi diff --git a/evergreen/wiki_page.sh b/evergreen/wiki_page.sh index fdf9ed27fde..6e9fd6317ba 100644 --- a/evergreen/wiki_page.sh +++ b/evergreen/wiki_page.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" set -o errexit @@ -10,4 +10,4 @@ $python -c 'import json; print(json.dumps([{ "link": "https://github.com/mongodb/mongo/wiki/Running-Tests-from-Evergreen-Tasks-Locally", "visibility": "public", "ignore_for_fetch": True -}]))' > wiki_page_location.json +}]))' >wiki_page_location.json diff --git a/evergreen/wiki_page_minimized_agg_query_fuzzer.sh b/evergreen/wiki_page_minimized_agg_query_fuzzer.sh index 5aa63fd89f1..e4c7ee38f35 100644 --- a/evergreen/wiki_page_minimized_agg_query_fuzzer.sh +++ b/evergreen/wiki_page_minimized_agg_query_fuzzer.sh @@ -1,15 +1,15 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" set -o errexit set -o verbose if [ -f "src/jstestfuzz/out/minimizer-outputs-minimizedtest.js" ]; then - activate_venv - $python -c 'import json; print(json.dumps([{ + activate_venv + $python -c 'import json; print(json.dumps([{ "name": "Wiki: Running minimized Agg fuzzer and Query fuzzer tests locally", "link": "https://github.com/mongodb/mongo/wiki/Running-minimized-Agg-fuzzer-and-Query-fuzzer-tests-locally", "visibility": "public", "ignore_for_fetch": True - }]))' > wiki_page_running_minimized_test_location.json + }]))' >wiki_page_running_minimized_test_location.json fi diff --git a/evergreen/write_mongo_binary_url_to_downstream_expansions.sh b/evergreen/write_mongo_binary_url_to_downstream_expansions.sh index 4ef24c2c507..9dc07f57d8c 100644 --- a/evergreen/write_mongo_binary_url_to_downstream_expansions.sh +++ b/evergreen/write_mongo_binary_url_to_downstream_expansions.sh @@ -1,4 +1,4 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" touch downstream_expansions.yaml diff --git a/evergreen/write_sast_report_env_file.sh b/evergreen/write_sast_report_env_file.sh index 5c3b9e12c1c..9f79bd59f06 100755 --- a/evergreen/write_sast_report_env_file.sh +++ b/evergreen/write_sast_report_env_file.sh @@ -1,6 +1,6 @@ #!/bin/env bash -cat << EOF > ${WORK_DIR}/sast_report_generation_credentials.env +cat <${WORK_DIR}/sast_report_generation_credentials.env COVERITY_USERNAME=${SAST_REPORT_COVERITY_USERNAME} COVERITY_PASSWORD=${SAST_REPORT_COVERITY_PASSWORD} GOOGLE_CLIENT_ID=${SAST_REPORT_UPLOAD_GOOGLE_CLIENT_ID} diff --git a/jstests/ssl/libs/localhost-cn-with-san.pem.sh b/jstests/ssl/libs/localhost-cn-with-san.pem.sh index 97f6273cf78..1b87e1e8cd3 100755 --- a/jstests/ssl/libs/localhost-cn-with-san.pem.sh +++ b/jstests/ssl/libs/localhost-cn-with-san.pem.sh @@ -8,15 +8,15 @@ OPENSSL="/opt/mongodbtoolchain/v4/bin/openssl" FILE="jstests/ssl/libs/localhost-cn-with-san" $OPENSSL req -new -subj "${RDN}" \ - -keyout "${FILE}.key" -out "${FILE}.csr" \ - -nodes -batch -sha256 -newkey rsa:2048 + -keyout "${FILE}.key" -out "${FILE}.csr" \ + -nodes -batch -sha256 -newkey rsa:2048 $OPENSSL rsa -in "${FILE}.key" -out "${FILE}.rsa" $OPENSSL x509 -in "${FILE}.csr" -out "${FILE}.pem" -req -CA "jstests/libs/ca.pem" \ - -days 3650 -CAcreateserial \ - -extfile <(printf "subjectAltName=DNS:example.com") + -days 3650 -CAcreateserial \ + -extfile <(printf "subjectAltName=DNS:example.com") # Create final bundle and cleanup. -cat "${FILE}.rsa" >> "${FILE}.pem" +cat "${FILE}.rsa" >>"${FILE}.pem" rm jstests/libs/ca.srl rm "${FILE}.key" "${FILE}.rsa" "${FILE}.csr" diff --git a/jstests/ssl/x509/mkcrl.sh b/jstests/ssl/x509/mkcrl.sh index da03ef4a24a..77b35015fa4 100755 --- a/jstests/ssl/x509/mkcrl.sh +++ b/jstests/ssl/x509/mkcrl.sh @@ -7,7 +7,7 @@ OUTPUT_PATH="jstests/libs/" CA_PEM_PATH="${OUTPUT_PATH}/ca.pem" TRUSTED_CA_PEM_PATH="${OUTPUT_PATH}/trusted-ca.pem" -die () { +die() { [ $# -gt 0 ] && [ ! -z "$1" ] && echo "$1" >&2 exit 1 } @@ -24,11 +24,11 @@ crl() { CA="$1" CONFIG="${CADB}/config" DEST="${OUTPUT_PATH}/$2" - echo '01' > "$CADB/serial" + echo '01' >"$CADB/serial" touch "$CADB/index.txt" "$CADB/index.txt.attr" - echo -e "[ ca ]\ndefault_ca = CA_default\n" > "$CONFIG" - echo -e "[ CA_default ]\ndatabase = ${CADB}/index.txt\n" >> "$CONFIG" - echo -e "certificate = $CA\nprivate_key = $CA\ndefault_md = sha256" >> "$CONFIG" + echo -e "[ ca ]\ndefault_ca = CA_default\n" >"$CONFIG" + echo -e "[ CA_default ]\ndatabase = ${CADB}/index.txt\n" >>"$CONFIG" + echo -e "certificate = $CA\nprivate_key = $CA\ndefault_md = sha256" >>"$CONFIG" VALIDITY_OPTIONS="-days 824 -crldays 823" if [ "$3" = "expired" ]; then diff --git a/modules_poc/private_headers.sh b/modules_poc/private_headers.sh index 76db7eb6b5e..c89dd1a8677 100755 --- a/modules_poc/private_headers.sh +++ b/modules_poc/private_headers.sh @@ -3,13 +3,11 @@ SCRIPT_DIR=$(dirname $0) SOURCE_ROOT="$SCRIPT_DIR/.." -function merged_decls_headers -{ +function merged_decls_headers { jq -r 'map(select(.mod as $mod | any(.used_from[]; .mod != $mod)) | .loc | split(":") | .[0]) | unique[]' $SOURCE_ROOT/merged_decls.json } -function all_headers -{ +function all_headers { cd $SOURCE_ROOT && find src/mongo -name '*.h' | sort } diff --git a/src/mongo/crypto/scripts/aead_encryption_fle2_test_vectors.sh b/src/mongo/crypto/scripts/aead_encryption_fle2_test_vectors.sh index bdd37298121..9e82085b87a 100755 --- a/src/mongo/crypto/scripts/aead_encryption_fle2_test_vectors.sh +++ b/src/mongo/crypto/scripts/aead_encryption_fle2_test_vectors.sh @@ -18,7 +18,7 @@ openssl enc -aes-256-${MODE:-ctr} -in M.bin -K $(xxd -c 100 -p Ke.bin) -iv $(xxd # echo -n "what do ya want for nothing?" | openssl dgst -sha256 -mac hmac -macopt hexkey:4a656665 -hex cat AD.bin IV.bin S.bin | openssl dgst -sha256 -mac hmac -macopt hexkey:$(xxd -c 100 -p Km.bin) -binary -out T.bin -cat IV.bin S.bin T.bin > C.bin +cat IV.bin S.bin T.bin >C.bin cat <&2 - exit 1 + echo "Usage: $0 SPEC_DIRECTORY" >&2 + exit 1 fi if ! [ -e "$1" ]; then - echo "Directory $1 not found" >&2 - exit 1 + echo "Directory $1 not found" >&2 + exit 1 fi if ! [ -d "$1" ]; then - echo "$1 not a directory" >&2 - exit 1 + echo "$1 not a directory" >&2 + exit 1 fi if ! [ -f "tla2tools.jar" ]; then - echo "No tla2tools.jar, run download-tlc.sh first" - exit 1 + echo "No tla2tools.jar, run download-tlc.sh first" + exit 1 fi TLA_FILE="MC$(echo $1 | sed 's/.*\///').tla" if ! [ -f "$1/$TLA_FILE" ]; then - echo "$1/$TLA_FILE does not exist" >&2 - exit 1 + echo "$1/$TLA_FILE does not exist" >&2 + exit 1 fi if [ -z "$JAVA_BINARY" ]; then - JAVA_BINARY=java + JAVA_BINARY=java else - echo "Using java binary [$JAVA_BINARY]" + echo "Using java binary [$JAVA_BINARY]" fi cd "$1"