mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-111537 Create an Evergreen task that runs multiple bazel-based resmoke tests (#44986) 

GitOrigin-RevId: cfe5a368311f304f71957b8022b792d01870618c
This commit is contained in:
Sean Lyons 2025-12-10 16:16:51 -05:00 committed by MongoDB Bot
parent f23d7f6b80
commit 9f5d68a902
12 changed files with 300 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.bazelrc
View File

@ -420,7 +420,6 @@ common:remote_test --remote_download_outputs=minimal
common:remote_test --test_output=summary
common:remote_test --modify_execution_info=^(CppLink|CppArchive|SolibSymlink|ExtractDebugInfo|StripDebugInfo|CcGenerateIntermediateDwp|CcGenerateDwp)$=-no-remote-cache
common:remote_test --remote_download_regex=.*(TestLogs.*|report.*\.json$|\.core$|data_archives.*\.tgz$) # Resmoke TestLogs directory, report, core dumps, and data directory archives.
test:remote_test --test_timeout=660 # Allow extra 60s for coredump on abort
test:remote_test --test_tag_filters=-incompatible_with_bazel_remote_test
# Coverage

12
bazel/resmoke/BUILD.bazel
View File

@ -80,6 +80,18 @@ config_setting(
},
)
bool_flag(
name = "installed_dist_test",
build_setting_default = False,
)
config_setting(
name = "installed_dist_test_enabled",
flag_values = {
"//bazel/resmoke:installed_dist_test": "True",
},
)
py_binary(
name = "resmoke_config_generator",
srcs = ["resmoke_config_generator.py"],

24
bazel/resmoke/resmoke.bzl
View File

@ -93,14 +93,20 @@ def resmoke_suite_test(
"--log=evg",
"--cedarReportFile=cedar_report.json",
"--skipSymbolization", # Symbolization is not yet functional, SERVER-103538
],
"//conditions:default": [],
}) + select({
"//bazel/resmoke:installed_dist_test_enabled": [
"--installDir=dist-test/bin",
"--mongoVersionFile=$(location //:.resmoke_mongo_version.yml)",
],
"//conditions:default": [
"--installDir=install-dist-test/bin",
"--mongoVersionFile=$(location //bazel/resmoke:resmoke_mongo_version)",
],
})
deps_path = ":".join(["$(location %s)" % dep for dep in deps])
native.py_test(
name = name,
# To a user of resmoke_suite_test, the `srcs` is the list of tests to select. However, to the py_test rule,
@ -122,13 +128,16 @@ def resmoke_suite_test(
"//:generated_resmoke_config",
"//:empty_jsconfig",
] + select({
"//bazel/resmoke:in_evergreen_enabled": ["//:installed-dist-test", "//:.resmoke_mongo_version.yml"],
"//conditions:default": ["//:install-dist-test", "//bazel/resmoke:resmoke_mongo_version"],
"//bazel/resmoke:installed_dist_test_enabled": ["//:installed-dist-test", "//:.resmoke_mongo_version.yml"],
"//conditions:default": ["//bazel/resmoke:resmoke_mongo_version"],
}),
deps = deps + [
deps = [
resmoke,
"//buildscripts:bazel_local_resources",
],
] + select({
"//bazel/resmoke:installed_dist_test_enabled": [],
"//conditions:default": deps,
}),
main = resmoke_shim,
args = [
"run",
@ -145,6 +154,9 @@ def resmoke_suite_test(
env = {
"LOCAL_RESOURCES": "$(LOCAL_RESOURCES)",
"GIT_PYTHON_REFRESH": "quiet", # Ignore "Bad git executable" error when importing git python. Git commands will still error if run.
},
} | select({
"//bazel/resmoke:installed_dist_test_enabled": {},
"//conditions:default": {"DEPS_PATH": deps_path},
}),
**kwargs
)

9
bazel/resmoke/resmoke_shim.py
View File

@ -105,6 +105,15 @@ if __name__ == "__main__":
add_evergreen_build_info(resmoke_args)
if os.environ.get("DEPS_PATH"):
# Modify DEPS_PATH to use os.pathsep, rather than ':'
os.environ["PATH"] += os.pathsep + os.pathsep.join(
[
os.path.dirname(os.path.abspath(path))
for path in os.environ.get("DEPS_PATH").split(":")
]
)
if os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR"):
undeclared_output_dir = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR")
resmoke_args.append(f"--dbpathPrefix={os.path.join(undeclared_output_dir,'data')}")

32
buildscripts/combine_reports.py
View File

@ -4,6 +4,7 @@
import errno
import json
import os
import re
import sys
from optparse import OptionParser
@ -46,6 +47,25 @@ def check_error(input_count, output_count):
raise ValueError("Both input file and output files exist")
def add_bazel_target_info(test_report, report_file):
outputs_path = os.path.dirname(
os.path.dirname(os.path.relpath(report_file, start="bazel-testlogs"))
)
if re.search(r"shard_\d+_of_\d+", os.path.basename(outputs_path)):
target_path = os.path.dirname(outputs_path)
else:
target_path = outputs_path
target = "//" + ":".join(target_path.rsplit("/", 1))
target_string = target.replace("/", "_").replace(":", "_")
for test in test_report.test_infos:
test.test_file = f"{target} - {test.test_file}"
test.group_id = f"{target_string}_{test.group_id}"
test.log_info["log_name"] = os.path.join(outputs_path, test.log_info["log_name"])
test.log_info["logs_to_merge"] = [
os.path.join(outputs_path, log) for log in test.log_info["logs_to_merge"]
]
def main():
"""Execute Main program."""
usage = "usage: %prog [options] report1.json report2.json ..."
@ -69,6 +89,13 @@ def main():
action="store_false",
help="Do not exit with a non-zero code if any test in the report fails.",
)
parser.add_option(
"--add-bazel-target-info",
dest="add_bazel_target_info",
default=False,
action="store_true",
help="Add bazel targets to the test names and log locations.",
)
(options, args) = parser.parse_args()
@ -82,7 +109,10 @@ def main():
for report_file in report_files:
try:
report_file_json = read_json_file(report_file)
test_reports.append(report.TestReport.from_dict(report_file_json))
test_report = report.TestReport.from_dict(report_file_json)
if options.add_bazel_target_info:
add_bazel_target_info(test_report, report_file)
test_reports.append(test_report)
except IOError as err:
# errno.ENOENT is the error code for "No such file or directory".
if err.errno == errno.ENOENT:

20
buildscripts/resmokeconfig/BUILD.bazel
View File

@ -52,8 +52,14 @@ resmoke_suite_test(
resmoke_args = [
"--storageEngineCacheSizeGB=1",
],
shard_count = 24,
tags = [
"manual", # exclude from expansion of target pattern wildcards (..., :*, :all, etc.)
"ci-development-critical-single-variant",
],
deps = [
"//src/mongo/db:mongod",
"//src/mongo/s:mongos",
"//src/mongo/shell:mongo",
],
)
@ -75,6 +81,10 @@ resmoke_suite_test(
tags = [
"manual", # exclude from expansion of target pattern wildcards (..., :*, :all, etc.)
],
deps = [
"//src/mongo/db:mongod",
"//src/mongo/shell:mongo",
],
)
# This is an experimental test target for running the multiversion_sanity_check
@ -100,6 +110,10 @@ resmoke_suite_test(
tags = [
"manual", # exclude from expansion of target pattern wildcards (..., :*, :all, etc.)
],
deps = [
"//src/mongo/db:mongod",
"//src/mongo/shell:mongo",
],
)
# This is an experimental test target for running the multiversion_sanity_check
@ -125,4 +139,8 @@ resmoke_suite_test(
tags = [
"manual", # exclude from expansion of target pattern wildcards (..., :*, :all, etc.)
],
deps = [
"//src/mongo/db:mongod",
"//src/mongo/shell:mongo",
],
)

28
etc/evergreen_yml_components/definitions.yml
View File

@ -1331,9 +1331,9 @@ functions:
OTEL_PARENT_ID: ${otel_parent_id}
OTEL_COLLECTOR_DIR: "../build/OTelTraces/"
"execute resmoke tests via bazel": &execute_resmoke_tests_via_bazel
"execute resmoke tests via bazel sh": &execute_resmoke_tests_via_bazel_sh
command: subprocess.exec
display_name: "execute resmoke tests via bazel"
display_name: "execute resmoke tests via bazel sh"
type: test
params:
binary: bash
@ -1352,6 +1352,12 @@ functions:
args:
- "./src/evergreen/resmoke_tests_execute_bazel.sh"
"execute resmoke tests via bazel":
- *get_version_expansions
- *apply_version_expansions
- *f_expansions_write
- *execute_resmoke_tests_via_bazel_sh
"assume ECR role": &assume_ecr_role
command: ec2.assume_role
params:
@ -1973,7 +1979,7 @@ functions:
- *update_task_timeout
- *f_expansions_write
- *sign_macos_dev_binaries
- *execute_resmoke_tests_via_bazel
- *execute_resmoke_tests_via_bazel_sh
"run generated tests via bazel":
- *f_expansions_write
@ -1997,7 +2003,7 @@ functions:
- *get_version_expansions
- *apply_version_expansions
- *f_expansions_write
- *execute_resmoke_tests_via_bazel
- *execute_resmoke_tests_via_bazel_sh
"generate version expansions": &generate_version_expansions
command: subprocess.exec
@ -3236,6 +3242,20 @@ functions:
remote_file: ${project}/${build_variant}/${revision}/datafiles/${task_id}-
display_name: "Data files "
"upload bazel test logs":
command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
bucket: mciuploads
permissions: private
visibility: signed
content_type: text/plain
local_files_include_filter:
- tmp/bazel-testlogs/*.log
remote_file: ${project}/${build_variant}/${revision}/${task_id}-
display_name: "Bazel test.log: "
"attach wiki page":
- *f_expansions_write
- command: subprocess.exec

95
etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml
View File

@ -486,40 +486,83 @@ tasks:
suite: //buildscripts/resmokeconfig:core
compiling_for_test: true
bazel_args: >-
--test_sharding_strategy=forced=16
--test_arg=--testTimeout=960
--test_timeout=1500
# Timeouts are 16 minutes for individual tests, the same as the default idle timeout.
# The bazel test timeout is 25 minutes, slightly shorter than the 30 minute commit queue
# timeout, so that it will timeout before the task would.
# Experimental task running the jstestfuzz suite bazel target. To be removed with SERVER-103537.
- <<: *jstestfuzz_template
name: bazel_jstestfuzz_gen
- name: resmoke_tests
tags: ["assigned_to_jira_team_devprod_correctness", "experimental"]
depends_on:
- name: version_expansions_gen
variant: generate-tasks-for-version
exec_timeout_secs: 1800 # 30 minutes.
commands:
- func: "generate resmoke tasks"
- func: "execute resmoke tests via bazel"
vars:
<<: *jstestfuzz_config_vars
num_files: 8
num_tasks: 2
jstestfuzz_vars: --jsTestsDir ../jstests
suite: //buildscripts/resmokeconfig:jstestfuzz
npm_command: jstestfuzz
targets: //buildscripts/...
compiling_for_test: true
bazel_args: >-
--test_tag_filters=${resmoke_tests_tag_filter},-incompatible_with_bazel_remote_test
--test_arg=--testTimeout=960
--test_timeout=1500
# Timeouts are 16 minutes for individual tests, the same as the default idle timeout.
# The bazel test timeout is 25 minutes, slightly shorter than the 30 minute exec
# timeout, so that it will timeout before the task would.
# The task_compile_flags are taken from what is set for archive_dist_test in bazel_compile.sh
task_compile_flags: >-
--verbose_failures
--simple_build_id=True
--features=strip_debug
--separate_debug=False
--define=MONGO_VERSION=${version}
--config=evg
--config=opt_profiled
--jobs=1600
# Experimental task running the multiversion sanity check bazel targets. To be removed with SERVER-103537.
- <<: *gen_task_template
name: bazel_multiversion_sanity_check_gen
tags:
[
"assigned_to_jira_team_devprod_correctness",
"experimental",
"multiversion",
"future_git_tag_incompatible",
]
commands:
- func: "generate resmoke tasks"
- func: "initialize multiversion tasks"
task_groups:
- name: resmoke_tests_TG
max_hosts: -1
setup_task_can_fail_task: true
setup_task:
- command: manifest.load
- func: "git get project and add git tag"
- func: "set task expansion macros"
- func: "f_expansions_write"
- func: "kill processes"
- func: "cleanup environment"
- func: "set up venv"
- func: "upload pip requirements"
- func: "configure evergreen api credentials"
- func: "set up credentials"
- func: "get engflow creds"
teardown_task:
- func: "debug full disk"
- func: "attach bazel invocation"
- func: "upload bazel test logs"
- func: "attach report"
- func: "upload mongodatafiles"
- func: "attach multiversion download links"
- func: "kill processes"
- func: "save mongo coredumps"
- func: "generate hang analyzer tasks"
- func: "save disk statistics"
- func: "save system resource information"
- func: "remove files"
vars:
//buildscripts/resmokeconfig:bazel_multiversion_sanity_check_last_continuous_new_new_old: last_continuous
//buildscripts/resmokeconfig:bazel_multiversion_sanity_check_last_lts_new_new_old: last_lts
files: >-
src/resmoke_error_code
src/*.gcda.gcov
src/gcov-intermediate-files.tgz
src/*.core src/*.mdmp src/*.core.gz src/*.mdmp.gz
mongo-coredumps.json
src/debugger*.*
src/mongo-hanganalyzer.tgz
diskstats.tgz
system-resource-info.tgz
${report_file|src/report.json}
${archive_file|src/archive.json}
src/network_diagnostics.txt
tasks:
- resmoke_tests

19
etc/evergreen_yml_components/variants/misc/misc_master_branch_only.yml
View File

@ -40,22 +40,15 @@ buildvariants:
cron: "0 4 * * 0" # Every week starting 0400 UTC Sunday
stepback: false
run_on:
- amazon2023-arm64-latest-m8g-xlarge
depends_on:
- name: archive_dist_test
variant: amazon-linux2023-arm64-static-compile
- name: archive_jstestshell
variant: amazon-linux2023-arm64-static-compile
- name: version_gen
variant: generate-tasks-for-version
- amazon2023-arm64-latest-m8g-4xlarge
expansions:
compile_variant: amazon-linux2023-arm64-static-compile
compile_variant: bazel-integration-tests
evergreen_remote_exec: on
multi_suite_resmoke_task: true
multiversion_platform: amazon2023
multiversion_edition: enterprise
multiversion_architecture: aarch64
multiversion_link_dir: multiversion_binaries
bazel_compile_flags: >-
--default_test_resources=cpu=1,2,4,8
resmoke_tests_tag_filter: ci-development-critical-single-variant
tasks:
- name: bazel_multiversion_sanity_check_gen
- name: bazel_jstestfuzz_gen
- name: resmoke_tests_TG

1
etc/expansions.default.yml
View File

@ -32,6 +32,7 @@ mciuploads_binary_visibility: public
jstestfuzz_concurrent_num_files: "10"
curator_release: "latest"
ext: tgz
resmoke_tests_tag_filter: resmoke_tests_tag_filter_not_set
# Bazel compile expansions.
compiling_for_test: "false"

1
evergreen/bazel_evergreen_shutils.sh
View File

@ -85,6 +85,7 @@ bazel_evergreen_shutils::compute_local_arg() {
local_arg+=" --jobs=auto"
elif [[ "$mode" == "test" && "${task_name:-}" == "unit_tests" ]]; then
local_arg+=" --config=remote_test"
local_arg+=" --test_timeout=660" # Allow extra 60s for coredump on abort
fi
if bazel_evergreen_shutils::is_ppc64le; then

182
evergreen/resmoke_tests_execute_bazel.sh
View File

@ -8,30 +8,29 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
. "$DIR/bazel_evergreen_shutils.sh"
set -o errexit
set -o verbose
activate_venv
source ./evergreen/bazel_evergreen_shutils.sh
bazel_evergreen_shutils::activate_and_cd_src
BAZEL_BINARY=$(bazel_evergreen_shutils::bazel_get_binary_path)
# Timeout is set here to avoid the build hanging indefinitely, still allowing
# for retries.
TIMEOUT_CMD=""
if [ -n "${build_timeout_seconds}" ]; then
TIMEOUT_CMD="timeout ${build_timeout_seconds}"
fi
ci_flags="\
if [ -z "${multi_suite_resmoke_task}" ]; then
ci_flags="\
--//bazel/resmoke:in_evergreen \
--//bazel/resmoke:installed_dist_test \
--test_output=all \
--noincompatible_enable_cc_toolchain_resolution \
--repo_env=no_c++_toolchain=1"
else
ci_flags="--//bazel/resmoke:in_evergreen"
# For simple build ID generation:
export compile_variant="${compile_variant}"
export version_id="${version_id}"
fi
if [[ "${evergreen_remote_exec}" == "on" ]]; then
ci_flags="--config=remote_test ${ci_flags}"
@ -62,74 +61,113 @@ for strategy in "${strategies[@]}"; do
ci_flags+=" --test_arg=--evergreenTestSelectionStrategy=${strategy}"
done
# If not explicitly specified on the target, pick a shard count that will fully utilize the current machine.
BUILD_INFO=$(bazel query ${targets} --output build)
if [[ "$BUILD_INFO" != *"shard_count ="* ]] && [[ "${bazel_args} ${bazel_compile_flags} ${task_compile_flags} ${patch_compile_flags}" != *"test_sharding_strategy"* ]]; then
CPUS=$(nproc)
SIZE=$(echo $BUILD_INFO | grep "size =" | cut -d '"' -f2)
TEST_RESOURCES_CPU=$(echo ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} ${patch_compile_flags} | awk -F'--default_test_resources=cpu=' '{print $2}')
TEST_RESOURCES_CPU=(${TEST_RESOURCES_CPU//,/ })
declare -A SIZES
SIZES=(["small"]=0 ["medium"]=1 ["large"]=2 ["enormous"]=3)
CPUS_PER_SHARD=${TEST_RESOURCES_CPU[${SIZES[$SIZE]}]}
SHARD_COUNT=$((CPUS / $CPUS_PER_SHARD))
ci_flags+=" --test_sharding_strategy=forced=$SHARD_COUNT"
fi
ALL_FLAGS="${ci_flags} ${LOCAL_ARG} ${bazel_args:-} ${bazel_compile_flags:-} ${task_compile_flags:-} ${patch_compile_flags:-}"
echo "${ALL_FLAGS}" >.bazel_build_flags
# Save the invocation, intentionally excluding CI specific flags.
echo "python buildscripts/install_bazel.py" >bazel-invocation.txt
echo "bazel test ${bazel_args} ${targets}" >>bazel-invocation.txt
set +o errexit
for i in {1..3}; do
eval ${TIMEOUT_CMD} ${BAZEL_BINARY} fetch ${ci_flags} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} ${patch_compile_flags} ${targets} && RET=0 && break || RET=$? && sleep 60
if [ $RET -eq 124 ]; then
echo "Bazel fetch timed out after ${build_timeout_seconds} seconds, retrying..."
else
echo "Bazel fetch failed, retrying..."
fi
$BAZEL_BINARY shutdown
done
# Save the invocation, intentionally excluding ci_flags.
echo "python buildscripts/install_bazel.py" >bazel-invocation.txt
echo "bazel test ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} ${patch_compile_flags} ${targets}" >>bazel-invocation.txt
eval ${BAZEL_BINARY} test ${ci_flags} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} ${patch_compile_flags} ${targets}
# Fetch then test with retries.
export RETRY_ON_FAIL=1
bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \
fetch ${ci_flags} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} ${patch_compile_flags} ${targets}
RET=$?
if [[ "$RET" == "0" ]]; then
export RETRY_ON_FAIL=0
bazel_evergreen_shutils::retry_bazel_cmd 3 "$BAZEL_BINARY" \
test ${ci_flags} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} ${patch_compile_flags} ${targets}
RET=$?
if [[ "$RET" -eq 124 ]]; then
echo "Bazel timed out after ${build_timeout_seconds:-<unspecified>} seconds."
elif [[ "$RET" != "0" ]]; then
echo "Errors were found during bazel test, failing the execution"
fi
fi
bazel_evergreen_shutils::write_last_engflow_link
set -o errexit
# Symlink data directories to where Resmoke normally puts them for compatability with post tasks
# that run for all Resmoke tasks.
find bazel-testlogs/ -path '*data/job*' -name 'job*' -print0 |
while IFS= read -r -d '' test_outputs; do
source=${workdir}/src/$test_outputs
target=${workdir}/$(sed 's/.*\.outputs\///' <<<$test_outputs)
mkdir -p $(dirname $target)
ln -sf $source $target
done
if [ -z "${multi_suite_resmoke_task}" ]; then
# Symlink data directories to where Resmoke normally puts them for compatibility with post tasks
# that run for all Resmoke tasks.
find bazel-testlogs/ -path '*data/job*' -name 'job*' -print0 |
while IFS= read -r -d '' test_outputs; do
source=${workdir}/src/$test_outputs
target=${workdir}/$(sed 's/.*\.outputs\///' <<<$test_outputs)
mkdir -p $(dirname $target)
ln -sf $source $target
done
# Symlink test logs to where Evergreen expects them. Evergreen won't read into a symlinked directory,
# so symlink each log file individually.
find bazel-testlogs/ -type f -path "*TestLogs/*" -print0 |
while IFS= read -r -d '' test_outputs; do
source=${workdir}/src/$test_outputs
target=${workdir}/$(sed 's/.*\.outputs\///' <<<$test_outputs)
mkdir -p $(dirname $target)
ln -sf $source $target
done
# Symlink test logs to where Evergreen expects them. Evergreen won't read into a symlinked directory,
# so symlink each log file individually.
find bazel-testlogs/ -type f -path "*TestLogs/*" -print0 |
while IFS= read -r -d '' test_outputs; do
source=${workdir}/src/$test_outputs
target=${workdir}/$(sed 's/.*\.outputs\///' <<<$test_outputs)
mkdir -p $(dirname $target)
ln -sf $source $target
done
# Symlinks archived data directories from multiple tests/shards to a single folder. Evergreen needs a
# single folder it can glob for s3.put. See the Evergreen function "upload mongodatafiles".
find bazel-testlogs/ -path '*data_archives/*.tgz' -print0 |
while IFS= read -r -d '' archive; do
source=${workdir}/src/$archive
target=${workdir}/$(sed 's/.*\.outputs\///' <<<$archive)
echo $source
echo $target
mkdir -p $(dirname $target)
ln -sf $source $target
done
# Symlinks archived data directories from multiple tests/shards to a single folder. Evergreen needs a
# single folder it can glob for s3.put. See the Evergreen function "upload mongodatafiles".
find bazel-testlogs/ -path '*data_archives/*.tgz' -print0 |
while IFS= read -r -d '' archive; do
source=${workdir}/src/$archive
target=${workdir}/$(sed 's/.*\.outputs\///' <<<$archive)
echo $source
echo $target
mkdir -p $(dirname $target)
ln -sf $source $target
done
# Combine reports from potentially multiple tests/shards.
find bazel-testlogs/ -name report*.json | xargs $python buildscripts/combine_reports.py --no-report-exit -o report.json
# Combine reports from potentially multiple tests/shards.
find bazel-testlogs/ -name report*.json | xargs $python buildscripts/combine_reports.py --no-report-exit -o report.json
else
# Symlink data directories to where Resmoke normally puts them for compatibility with post tasks
# that run for all Resmoke tasks.
find bazel-testlogs/ -path '*data/job*' -name 'job*' -print0 |
while IFS= read -r -d '' test_outputs; do
source=${workdir}/src/$test_outputs
target=${workdir}/$(sed 's/.*\.outputs\///' <<<$test_outputs)
mkdir -p $(dirname $target)
ln -sf $source $target
done
# Symlinks archived data directories from multiple tests/shards to a single folder. Evergreen needs a
# single folder it can glob for s3.put. See the Evergreen function "upload mongodatafiles".
target_from_undeclared_outputs() {
echo ${1} | sed -e 's/^.*bazel-testlogs\/\(.*\)\/test.outputs.*$/\1/'
}
find bazel-testlogs/ -path '*data_archives/*.tgz' -print0 |
while IFS= read -r -d '' archive; do
source=${workdir}/src/$archive
bazel_target_prefix=$(target_from_undeclared_outputs $archive | sed 's/\//_/g')
target=${workdir}/$(echo $archive | sed -e 's/.*\.outputs\///' -e "s/data_archives\//&$bazel_target_prefix-/g")
mkdir -p $(dirname $target)
ln -sf $source $target
done
# Symlinks test.log from multiple tests/shards to a single folder. Evergreen needs a
# single folder it can glob for s3.put. See the Evergreen function "upload bazel test logs".
find bazel-testlogs/ -path '*test.log' -print0 |
while IFS= read -r -d '' log; do
log_renamed=$(echo $log | sed -e 's/bazel-testlogs\///g' -e 's/\//_/g')
source=${workdir}/src/$log
target=${workdir}/tmp/bazel-testlogs/$log_renamed
mkdir -p $(dirname $target)
ln -sf $source $target
done
echo "format: text-timestamp" >${workdir}/build/TestLogs/log_spec.yaml
echo "version: 0" >>${workdir}/build/TestLogs/log_spec.yaml
# Combine reports from potentially multiple tests/shards.
find bazel-testlogs/ -name report*.json | xargs $python buildscripts/combine_reports.py --no-report-exit --add-bazel-target-info -o report.json
fi
exit $RET