diff --git a/.bazelrc b/.bazelrc index aef6aec367d..ecebc903a9d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -420,7 +420,6 @@ common:remote_test --remote_download_outputs=minimal common:remote_test --test_output=summary common:remote_test --modify_execution_info=^(CppLink|CppArchive|SolibSymlink|ExtractDebugInfo|StripDebugInfo|CcGenerateIntermediateDwp|CcGenerateDwp)$=-no-remote-cache common:remote_test --remote_download_regex=.*(TestLogs.*|report.*\.json$|\.core$|data_archives.*\.tgz$) # Resmoke TestLogs directory, report, core dumps, and data directory archives. -test:remote_test --test_timeout=660 # Allow extra 60s for coredump on abort test:remote_test --test_tag_filters=-incompatible_with_bazel_remote_test # Coverage diff --git a/bazel/resmoke/BUILD.bazel b/bazel/resmoke/BUILD.bazel index cd83b43cef4..ea629311472 100644 --- a/bazel/resmoke/BUILD.bazel +++ b/bazel/resmoke/BUILD.bazel @@ -80,6 +80,18 @@ config_setting( }, ) +bool_flag( + name = "installed_dist_test", + build_setting_default = False, +) + +config_setting( + name = "installed_dist_test_enabled", + flag_values = { + "//bazel/resmoke:installed_dist_test": "True", + }, +) + py_binary( name = "resmoke_config_generator", srcs = ["resmoke_config_generator.py"], diff --git a/bazel/resmoke/resmoke.bzl b/bazel/resmoke/resmoke.bzl index ccfcdbb1abf..de70b88017d 100644 --- a/bazel/resmoke/resmoke.bzl +++ b/bazel/resmoke/resmoke.bzl @@ -93,14 +93,20 @@ def resmoke_suite_test( "--log=evg", "--cedarReportFile=cedar_report.json", "--skipSymbolization", # Symbolization is not yet functional, SERVER-103538 + ], + "//conditions:default": [], + }) + select({ + "//bazel/resmoke:installed_dist_test_enabled": [ "--installDir=dist-test/bin", "--mongoVersionFile=$(location //:.resmoke_mongo_version.yml)", ], "//conditions:default": [ - "--installDir=install-dist-test/bin", "--mongoVersionFile=$(location //bazel/resmoke:resmoke_mongo_version)", ], }) + + deps_path = ":".join(["$(location %s)" % dep for dep in deps]) + native.py_test( name = name, # To a user of resmoke_suite_test, the `srcs` is the list of tests to select. However, to the py_test rule, @@ -122,13 +128,16 @@ def resmoke_suite_test( "//:generated_resmoke_config", "//:empty_jsconfig", ] + select({ - "//bazel/resmoke:in_evergreen_enabled": ["//:installed-dist-test", "//:.resmoke_mongo_version.yml"], - "//conditions:default": ["//:install-dist-test", "//bazel/resmoke:resmoke_mongo_version"], + "//bazel/resmoke:installed_dist_test_enabled": ["//:installed-dist-test", "//:.resmoke_mongo_version.yml"], + "//conditions:default": ["//bazel/resmoke:resmoke_mongo_version"], }), - deps = deps + [ + deps = [ resmoke, "//buildscripts:bazel_local_resources", - ], + ] + select({ + "//bazel/resmoke:installed_dist_test_enabled": [], + "//conditions:default": deps, + }), main = resmoke_shim, args = [ "run", @@ -145,6 +154,9 @@ def resmoke_suite_test( env = { "LOCAL_RESOURCES": "$(LOCAL_RESOURCES)", "GIT_PYTHON_REFRESH": "quiet", # Ignore "Bad git executable" error when importing git python. Git commands will still error if run. - }, + } | select({ + "//bazel/resmoke:installed_dist_test_enabled": {}, + "//conditions:default": {"DEPS_PATH": deps_path}, + }), **kwargs ) diff --git a/bazel/resmoke/resmoke_shim.py b/bazel/resmoke/resmoke_shim.py index 0b44077db96..5e7ab1ea100 100644 --- a/bazel/resmoke/resmoke_shim.py +++ b/bazel/resmoke/resmoke_shim.py @@ -105,6 +105,15 @@ if __name__ == "__main__": add_evergreen_build_info(resmoke_args) + if os.environ.get("DEPS_PATH"): + # Modify DEPS_PATH to use os.pathsep, rather than ':' + os.environ["PATH"] += os.pathsep + os.pathsep.join( + [ + os.path.dirname(os.path.abspath(path)) + for path in os.environ.get("DEPS_PATH").split(":") + ] + ) + if os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR"): undeclared_output_dir = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR") resmoke_args.append(f"--dbpathPrefix={os.path.join(undeclared_output_dir,'data')}") diff --git a/buildscripts/combine_reports.py b/buildscripts/combine_reports.py index 75210478060..ebc937932b7 100755 --- a/buildscripts/combine_reports.py +++ b/buildscripts/combine_reports.py @@ -4,6 +4,7 @@ import errno import json import os +import re import sys from optparse import OptionParser @@ -46,6 +47,25 @@ def check_error(input_count, output_count): raise ValueError("Both input file and output files exist") +def add_bazel_target_info(test_report, report_file): + outputs_path = os.path.dirname( + os.path.dirname(os.path.relpath(report_file, start="bazel-testlogs")) + ) + if re.search(r"shard_\d+_of_\d+", os.path.basename(outputs_path)): + target_path = os.path.dirname(outputs_path) + else: + target_path = outputs_path + target = "//" + ":".join(target_path.rsplit("/", 1)) + target_string = target.replace("/", "_").replace(":", "_") + for test in test_report.test_infos: + test.test_file = f"{target} - {test.test_file}" + test.group_id = f"{target_string}_{test.group_id}" + test.log_info["log_name"] = os.path.join(outputs_path, test.log_info["log_name"]) + test.log_info["logs_to_merge"] = [ + os.path.join(outputs_path, log) for log in test.log_info["logs_to_merge"] + ] + + def main(): """Execute Main program.""" usage = "usage: %prog [options] report1.json report2.json ..." @@ -69,6 +89,13 @@ def main(): action="store_false", help="Do not exit with a non-zero code if any test in the report fails.", ) + parser.add_option( + "--add-bazel-target-info", + dest="add_bazel_target_info", + default=False, + action="store_true", + help="Add bazel targets to the test names and log locations.", + ) (options, args) = parser.parse_args() @@ -82,7 +109,10 @@ def main(): for report_file in report_files: try: report_file_json = read_json_file(report_file) - test_reports.append(report.TestReport.from_dict(report_file_json)) + test_report = report.TestReport.from_dict(report_file_json) + if options.add_bazel_target_info: + add_bazel_target_info(test_report, report_file) + test_reports.append(test_report) except IOError as err: # errno.ENOENT is the error code for "No such file or directory". if err.errno == errno.ENOENT: diff --git a/buildscripts/resmokeconfig/BUILD.bazel b/buildscripts/resmokeconfig/BUILD.bazel index be1362613b6..f47dcde69db 100644 --- a/buildscripts/resmokeconfig/BUILD.bazel +++ b/buildscripts/resmokeconfig/BUILD.bazel @@ -52,8 +52,14 @@ resmoke_suite_test( resmoke_args = [ "--storageEngineCacheSizeGB=1", ], + shard_count = 24, tags = [ - "manual", # exclude from expansion of target pattern wildcards (..., :*, :all, etc.) + "ci-development-critical-single-variant", + ], + deps = [ + "//src/mongo/db:mongod", + "//src/mongo/s:mongos", + "//src/mongo/shell:mongo", ], ) @@ -75,6 +81,10 @@ resmoke_suite_test( tags = [ "manual", # exclude from expansion of target pattern wildcards (..., :*, :all, etc.) ], + deps = [ + "//src/mongo/db:mongod", + "//src/mongo/shell:mongo", + ], ) # This is an experimental test target for running the multiversion_sanity_check @@ -100,6 +110,10 @@ resmoke_suite_test( tags = [ "manual", # exclude from expansion of target pattern wildcards (..., :*, :all, etc.) ], + deps = [ + "//src/mongo/db:mongod", + "//src/mongo/shell:mongo", + ], ) # This is an experimental test target for running the multiversion_sanity_check @@ -125,4 +139,8 @@ resmoke_suite_test( tags = [ "manual", # exclude from expansion of target pattern wildcards (..., :*, :all, etc.) ], + deps = [ + "//src/mongo/db:mongod", + "//src/mongo/shell:mongo", + ], ) diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index a17adc8dff2..6bd83735738 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -1331,9 +1331,9 @@ functions: OTEL_PARENT_ID: ${otel_parent_id} OTEL_COLLECTOR_DIR: "../build/OTelTraces/" - "execute resmoke tests via bazel": &execute_resmoke_tests_via_bazel + "execute resmoke tests via bazel sh": &execute_resmoke_tests_via_bazel_sh command: subprocess.exec - display_name: "execute resmoke tests via bazel" + display_name: "execute resmoke tests via bazel sh" type: test params: binary: bash @@ -1352,6 +1352,12 @@ functions: args: - "./src/evergreen/resmoke_tests_execute_bazel.sh" + "execute resmoke tests via bazel": + - *get_version_expansions + - *apply_version_expansions + - *f_expansions_write + - *execute_resmoke_tests_via_bazel_sh + "assume ECR role": &assume_ecr_role command: ec2.assume_role params: @@ -1973,7 +1979,7 @@ functions: - *update_task_timeout - *f_expansions_write - *sign_macos_dev_binaries - - *execute_resmoke_tests_via_bazel + - *execute_resmoke_tests_via_bazel_sh "run generated tests via bazel": - *f_expansions_write @@ -1997,7 +2003,7 @@ functions: - *get_version_expansions - *apply_version_expansions - *f_expansions_write - - *execute_resmoke_tests_via_bazel + - *execute_resmoke_tests_via_bazel_sh "generate version expansions": &generate_version_expansions command: subprocess.exec @@ -3236,6 +3242,20 @@ functions: remote_file: ${project}/${build_variant}/${revision}/datafiles/${task_id}- display_name: "Data files " + "upload bazel test logs": + command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + permissions: private + visibility: signed + content_type: text/plain + local_files_include_filter: + - tmp/bazel-testlogs/*.log + remote_file: ${project}/${build_variant}/${revision}/${task_id}- + display_name: "Bazel test.log: " + "attach wiki page": - *f_expansions_write - command: subprocess.exec diff --git a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml index e8d56578059..44dfda162c6 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml @@ -486,40 +486,83 @@ tasks: suite: //buildscripts/resmokeconfig:core compiling_for_test: true bazel_args: >- - --test_sharding_strategy=forced=16 --test_arg=--testTimeout=960 --test_timeout=1500 # Timeouts are 16 minutes for individual tests, the same as the default idle timeout. # The bazel test timeout is 25 minutes, slightly shorter than the 30 minute commit queue # timeout, so that it will timeout before the task would. - # Experimental task running the jstestfuzz suite bazel target. To be removed with SERVER-103537. - - <<: *jstestfuzz_template - name: bazel_jstestfuzz_gen + - name: resmoke_tests tags: ["assigned_to_jira_team_devprod_correctness", "experimental"] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + exec_timeout_secs: 1800 # 30 minutes. commands: - - func: "generate resmoke tasks" + - func: "execute resmoke tests via bazel" vars: - <<: *jstestfuzz_config_vars - num_files: 8 - num_tasks: 2 - jstestfuzz_vars: --jsTestsDir ../jstests - suite: //buildscripts/resmokeconfig:jstestfuzz - npm_command: jstestfuzz + targets: //buildscripts/... + compiling_for_test: true + bazel_args: >- + --test_tag_filters=${resmoke_tests_tag_filter},-incompatible_with_bazel_remote_test + --test_arg=--testTimeout=960 + --test_timeout=1500 + # Timeouts are 16 minutes for individual tests, the same as the default idle timeout. + # The bazel test timeout is 25 minutes, slightly shorter than the 30 minute exec + # timeout, so that it will timeout before the task would. + # The task_compile_flags are taken from what is set for archive_dist_test in bazel_compile.sh + task_compile_flags: >- + --verbose_failures + --simple_build_id=True + --features=strip_debug + --separate_debug=False + --define=MONGO_VERSION=${version} + --config=evg + --config=opt_profiled + --jobs=1600 - # Experimental task running the multiversion sanity check bazel targets. To be removed with SERVER-103537. - - <<: *gen_task_template - name: bazel_multiversion_sanity_check_gen - tags: - [ - "assigned_to_jira_team_devprod_correctness", - "experimental", - "multiversion", - "future_git_tag_incompatible", - ] - commands: - - func: "generate resmoke tasks" - - func: "initialize multiversion tasks" +task_groups: + - name: resmoke_tests_TG + max_hosts: -1 + setup_task_can_fail_task: true + setup_task: + - command: manifest.load + - func: "git get project and add git tag" + - func: "set task expansion macros" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "configure evergreen api credentials" + - func: "set up credentials" + - func: "get engflow creds" + teardown_task: + - func: "debug full disk" + - func: "attach bazel invocation" + - func: "upload bazel test logs" + - func: "attach report" + - func: "upload mongodatafiles" + - func: "attach multiversion download links" + - func: "kill processes" + - func: "save mongo coredumps" + - func: "generate hang analyzer tasks" + - func: "save disk statistics" + - func: "save system resource information" + - func: "remove files" vars: - //buildscripts/resmokeconfig:bazel_multiversion_sanity_check_last_continuous_new_new_old: last_continuous - //buildscripts/resmokeconfig:bazel_multiversion_sanity_check_last_lts_new_new_old: last_lts + files: >- + src/resmoke_error_code + src/*.gcda.gcov + src/gcov-intermediate-files.tgz + src/*.core src/*.mdmp src/*.core.gz src/*.mdmp.gz + mongo-coredumps.json + src/debugger*.* + src/mongo-hanganalyzer.tgz + diskstats.tgz + system-resource-info.tgz + ${report_file|src/report.json} + ${archive_file|src/archive.json} + src/network_diagnostics.txt + tasks: + - resmoke_tests diff --git a/etc/evergreen_yml_components/variants/misc/misc_master_branch_only.yml b/etc/evergreen_yml_components/variants/misc/misc_master_branch_only.yml index bba1e6cefe1..08921e0fb07 100644 --- a/etc/evergreen_yml_components/variants/misc/misc_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/misc/misc_master_branch_only.yml @@ -40,22 +40,15 @@ buildvariants: cron: "0 4 * * 0" # Every week starting 0400 UTC Sunday stepback: false run_on: - - amazon2023-arm64-latest-m8g-xlarge - depends_on: - - name: archive_dist_test - variant: amazon-linux2023-arm64-static-compile - - name: archive_jstestshell - variant: amazon-linux2023-arm64-static-compile - - name: version_gen - variant: generate-tasks-for-version + - amazon2023-arm64-latest-m8g-4xlarge expansions: - compile_variant: amazon-linux2023-arm64-static-compile + compile_variant: bazel-integration-tests + evergreen_remote_exec: on + multi_suite_resmoke_task: true multiversion_platform: amazon2023 multiversion_edition: enterprise multiversion_architecture: aarch64 multiversion_link_dir: multiversion_binaries - bazel_compile_flags: >- - --default_test_resources=cpu=1,2,4,8 + resmoke_tests_tag_filter: ci-development-critical-single-variant tasks: - - name: bazel_multiversion_sanity_check_gen - - name: bazel_jstestfuzz_gen + - name: resmoke_tests_TG diff --git a/etc/expansions.default.yml b/etc/expansions.default.yml index 7480216f788..5d5356ec4d9 100644 --- a/etc/expansions.default.yml +++ b/etc/expansions.default.yml @@ -32,6 +32,7 @@ mciuploads_binary_visibility: public jstestfuzz_concurrent_num_files: "10" curator_release: "latest" ext: tgz +resmoke_tests_tag_filter: resmoke_tests_tag_filter_not_set # Bazel compile expansions. compiling_for_test: "false" diff --git a/evergreen/bazel_evergreen_shutils.sh b/evergreen/bazel_evergreen_shutils.sh index 7c27c616b95..8e7025cb752 100644 --- a/evergreen/bazel_evergreen_shutils.sh +++ b/evergreen/bazel_evergreen_shutils.sh @@ -85,6 +85,7 @@ bazel_evergreen_shutils::compute_local_arg() { local_arg+=" --jobs=auto" elif [[ "$mode" == "test" && "${task_name:-}" == "unit_tests" ]]; then local_arg+=" --config=remote_test" + local_arg+=" --test_timeout=660" # Allow extra 60s for coredump on abort fi if bazel_evergreen_shutils::is_ppc64le; then diff --git a/evergreen/resmoke_tests_execute_bazel.sh b/evergreen/resmoke_tests_execute_bazel.sh index c42682174d6..ba65847a15b 100644 --- a/evergreen/resmoke_tests_execute_bazel.sh +++ b/evergreen/resmoke_tests_execute_bazel.sh @@ -8,30 +8,29 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" - -cd src +. "$DIR/bazel_evergreen_shutils.sh" set -o errexit set -o verbose -activate_venv - -source ./evergreen/bazel_evergreen_shutils.sh +bazel_evergreen_shutils::activate_and_cd_src BAZEL_BINARY=$(bazel_evergreen_shutils::bazel_get_binary_path) -# Timeout is set here to avoid the build hanging indefinitely, still allowing -# for retries. -TIMEOUT_CMD="" -if [ -n "${build_timeout_seconds}" ]; then - TIMEOUT_CMD="timeout ${build_timeout_seconds}" -fi - -ci_flags="\ +if [ -z "${multi_suite_resmoke_task}" ]; then + ci_flags="\ --//bazel/resmoke:in_evergreen \ +--//bazel/resmoke:installed_dist_test \ --test_output=all \ --noincompatible_enable_cc_toolchain_resolution \ --repo_env=no_c++_toolchain=1" +else + ci_flags="--//bazel/resmoke:in_evergreen" + + # For simple build ID generation: + export compile_variant="${compile_variant}" + export version_id="${version_id}" +fi if [[ "${evergreen_remote_exec}" == "on" ]]; then ci_flags="--config=remote_test ${ci_flags}" @@ -62,74 +61,113 @@ for strategy in "${strategies[@]}"; do ci_flags+=" --test_arg=--evergreenTestSelectionStrategy=${strategy}" done -# If not explicitly specified on the target, pick a shard count that will fully utilize the current machine. -BUILD_INFO=$(bazel query ${targets} --output build) -if [[ "$BUILD_INFO" != *"shard_count ="* ]] && [[ "${bazel_args} ${bazel_compile_flags} ${task_compile_flags} ${patch_compile_flags}" != *"test_sharding_strategy"* ]]; then - CPUS=$(nproc) - SIZE=$(echo $BUILD_INFO | grep "size =" | cut -d '"' -f2) - TEST_RESOURCES_CPU=$(echo ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} ${patch_compile_flags} | awk -F'--default_test_resources=cpu=' '{print $2}') - TEST_RESOURCES_CPU=(${TEST_RESOURCES_CPU//,/ }) - declare -A SIZES - SIZES=(["small"]=0 ["medium"]=1 ["large"]=2 ["enormous"]=3) - CPUS_PER_SHARD=${TEST_RESOURCES_CPU[${SIZES[$SIZE]}]} - SHARD_COUNT=$((CPUS / $CPUS_PER_SHARD)) - ci_flags+=" --test_sharding_strategy=forced=$SHARD_COUNT" -fi +ALL_FLAGS="${ci_flags} ${LOCAL_ARG} ${bazel_args:-} ${bazel_compile_flags:-} ${task_compile_flags:-} ${patch_compile_flags:-}" +echo "${ALL_FLAGS}" >.bazel_build_flags + +# Save the invocation, intentionally excluding CI specific flags. +echo "python buildscripts/install_bazel.py" >bazel-invocation.txt +echo "bazel test ${bazel_args} ${targets}" >>bazel-invocation.txt set +o errexit -for i in {1..3}; do - eval ${TIMEOUT_CMD} ${BAZEL_BINARY} fetch ${ci_flags} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} ${patch_compile_flags} ${targets} && RET=0 && break || RET=$? && sleep 60 - if [ $RET -eq 124 ]; then - echo "Bazel fetch timed out after ${build_timeout_seconds} seconds, retrying..." - else - echo "Bazel fetch failed, retrying..." - fi - $BAZEL_BINARY shutdown -done - -# Save the invocation, intentionally excluding ci_flags. -echo "python buildscripts/install_bazel.py" >bazel-invocation.txt -echo "bazel test ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} ${patch_compile_flags} ${targets}" >>bazel-invocation.txt - -eval ${BAZEL_BINARY} test ${ci_flags} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} ${patch_compile_flags} ${targets} +# Fetch then test with retries. +export RETRY_ON_FAIL=1 +bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \ + fetch ${ci_flags} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} ${patch_compile_flags} ${targets} RET=$? +if [[ "$RET" == "0" ]]; then + export RETRY_ON_FAIL=0 + bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \ + test ${ci_flags} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} ${patch_compile_flags} ${targets} + RET=$? + + if [[ "$RET" -eq 124 ]]; then + echo "Bazel timed out after ${build_timeout_seconds:-} seconds." + elif [[ "$RET" != "0" ]]; then + echo "Errors were found during bazel test, failing the execution" + fi +fi + +bazel_evergreen_shutils::write_last_engflow_link + set -o errexit -# Symlink data directories to where Resmoke normally puts them for compatability with post tasks -# that run for all Resmoke tasks. -find bazel-testlogs/ -path '*data/job*' -name 'job*' -print0 | - while IFS= read -r -d '' test_outputs; do - source=${workdir}/src/$test_outputs - target=${workdir}/$(sed 's/.*\.outputs\///' <<<$test_outputs) - mkdir -p $(dirname $target) - ln -sf $source $target - done +if [ -z "${multi_suite_resmoke_task}" ]; then + # Symlink data directories to where Resmoke normally puts them for compatibility with post tasks + # that run for all Resmoke tasks. + find bazel-testlogs/ -path '*data/job*' -name 'job*' -print0 | + while IFS= read -r -d '' test_outputs; do + source=${workdir}/src/$test_outputs + target=${workdir}/$(sed 's/.*\.outputs\///' <<<$test_outputs) + mkdir -p $(dirname $target) + ln -sf $source $target + done -# Symlink test logs to where Evergreen expects them. Evergreen won't read into a symlinked directory, -# so symlink each log file individually. -find bazel-testlogs/ -type f -path "*TestLogs/*" -print0 | - while IFS= read -r -d '' test_outputs; do - source=${workdir}/src/$test_outputs - target=${workdir}/$(sed 's/.*\.outputs\///' <<<$test_outputs) - mkdir -p $(dirname $target) - ln -sf $source $target - done + # Symlink test logs to where Evergreen expects them. Evergreen won't read into a symlinked directory, + # so symlink each log file individually. + find bazel-testlogs/ -type f -path "*TestLogs/*" -print0 | + while IFS= read -r -d '' test_outputs; do + source=${workdir}/src/$test_outputs + target=${workdir}/$(sed 's/.*\.outputs\///' <<<$test_outputs) + mkdir -p $(dirname $target) + ln -sf $source $target + done -# Symlinks archived data directories from multiple tests/shards to a single folder. Evergreen needs a -# single folder it can glob for s3.put. See the Evergreen function "upload mongodatafiles". -find bazel-testlogs/ -path '*data_archives/*.tgz' -print0 | - while IFS= read -r -d '' archive; do - source=${workdir}/src/$archive - target=${workdir}/$(sed 's/.*\.outputs\///' <<<$archive) - echo $source - echo $target - mkdir -p $(dirname $target) - ln -sf $source $target - done + # Symlinks archived data directories from multiple tests/shards to a single folder. Evergreen needs a + # single folder it can glob for s3.put. See the Evergreen function "upload mongodatafiles". + find bazel-testlogs/ -path '*data_archives/*.tgz' -print0 | + while IFS= read -r -d '' archive; do + source=${workdir}/src/$archive + target=${workdir}/$(sed 's/.*\.outputs\///' <<<$archive) + echo $source + echo $target + mkdir -p $(dirname $target) + ln -sf $source $target + done -# Combine reports from potentially multiple tests/shards. -find bazel-testlogs/ -name report*.json | xargs $python buildscripts/combine_reports.py --no-report-exit -o report.json + # Combine reports from potentially multiple tests/shards. + find bazel-testlogs/ -name report*.json | xargs $python buildscripts/combine_reports.py --no-report-exit -o report.json +else + # Symlink data directories to where Resmoke normally puts them for compatibility with post tasks + # that run for all Resmoke tasks. + find bazel-testlogs/ -path '*data/job*' -name 'job*' -print0 | + while IFS= read -r -d '' test_outputs; do + source=${workdir}/src/$test_outputs + target=${workdir}/$(sed 's/.*\.outputs\///' <<<$test_outputs) + mkdir -p $(dirname $target) + ln -sf $source $target + done + + # Symlinks archived data directories from multiple tests/shards to a single folder. Evergreen needs a + # single folder it can glob for s3.put. See the Evergreen function "upload mongodatafiles". + target_from_undeclared_outputs() { + echo ${1} | sed -e 's/^.*bazel-testlogs\/\(.*\)\/test.outputs.*$/\1/' + } + find bazel-testlogs/ -path '*data_archives/*.tgz' -print0 | + while IFS= read -r -d '' archive; do + source=${workdir}/src/$archive + bazel_target_prefix=$(target_from_undeclared_outputs $archive | sed 's/\//_/g') + target=${workdir}/$(echo $archive | sed -e 's/.*\.outputs\///' -e "s/data_archives\//&$bazel_target_prefix-/g") + mkdir -p $(dirname $target) + ln -sf $source $target + done + + # Symlinks test.log from multiple tests/shards to a single folder. Evergreen needs a + # single folder it can glob for s3.put. See the Evergreen function "upload bazel test logs". + find bazel-testlogs/ -path '*test.log' -print0 | + while IFS= read -r -d '' log; do + log_renamed=$(echo $log | sed -e 's/bazel-testlogs\///g' -e 's/\//_/g') + source=${workdir}/src/$log + target=${workdir}/tmp/bazel-testlogs/$log_renamed + mkdir -p $(dirname $target) + ln -sf $source $target + done + echo "format: text-timestamp" >${workdir}/build/TestLogs/log_spec.yaml + echo "version: 0" >>${workdir}/build/TestLogs/log_spec.yaml + + # Combine reports from potentially multiple tests/shards. + find bazel-testlogs/ -name report*.json | xargs $python buildscripts/combine_reports.py --no-report-exit --add-bazel-target-info -o report.json +fi exit $RET