SERVER-101034 Use rules_lint shfmt formatter (#38448)

GitOrigin-RevId: e8ef1ba2000e12fa2cd5a115a9ceeab92332e938
This commit is contained in:
Zack Winter 2025-07-15 18:19:08 -07:00 committed by MongoDB Bot
parent 5ac113adb1
commit 7ccc14bf91
174 changed files with 2259 additions and 2387 deletions

9
.editorconfig Normal file
View File

@ -0,0 +1,9 @@
root = true
[*.sh]
charset = utf-8
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
indent_style = space
indent_size = 4

1
.github/CODEOWNERS vendored
View File

@ -10,6 +10,7 @@ OWNERS.yml @10gen/server-root-ownership @svc-auto-approve-bot
.bazel* @10gen/devprod-build @svc-auto-approve-bot .bazel* @10gen/devprod-build @svc-auto-approve-bot
.clang-format @10gen/server-programmability @svc-auto-approve-bot .clang-format @10gen/server-programmability @svc-auto-approve-bot
.clang-tidy.in @10gen/server-programmability @svc-auto-approve-bot .clang-tidy.in @10gen/server-programmability @svc-auto-approve-bot
/.editorconfig @10gen/devprod-build @svc-auto-approve-bot
.git* @10gen/devprod-build @svc-auto-approve-bot .git* @10gen/devprod-build @svc-auto-approve-bot
.mypy.ini @10gen/devprod-build @10gen/devprod-correctness @svc-auto-approve-bot .mypy.ini @10gen/devprod-build @10gen/devprod-correctness @svc-auto-approve-bot
.prettierignore @10gen/devprod-correctness @svc-auto-approve-bot .prettierignore @10gen/devprod-correctness @svc-auto-approve-bot

View File

@ -18,6 +18,9 @@ filters:
- ".clang-tidy.in": - ".clang-tidy.in":
approvers: approvers:
- 10gen/server-programmability - 10gen/server-programmability
- "/.editorconfig":
approvers:
- 10gen/devprod-build
- ".git*": - ".git*":
approvers: approvers:
- 10gen/devprod-build - 10gen/devprod-build

View File

@ -6,8 +6,6 @@ py_binary(
args = [ args = [
"--prettier", "--prettier",
"$(location //:prettier)", "$(location //:prettier)",
"--shellscripts-linters",
"$(location //buildscripts:shellscripts_linters)",
"--rules-lint-format", "--rules-lint-format",
"$(location :rules_lint_format)", "$(location :rules_lint_format)",
"--rules-lint-format-check", "--rules-lint-format-check",
@ -17,7 +15,6 @@ py_binary(
":rules_lint_format", ":rules_lint_format",
":rules_lint_format.check", ":rules_lint_format.check",
"//:prettier", "//:prettier",
"//buildscripts:shellscripts_linters",
"@shfmt", "@shfmt",
], ],
env = { env = {
@ -38,9 +35,8 @@ format_multirun(
graphql = "//:prettier", graphql = "//:prettier",
html = "//:prettier", html = "//:prettier",
markdown = "//:prettier", markdown = "//:prettier",
shell = "@shfmt//:shfmt",
sql = "//:prettier", sql = "//:prettier",
starlark = "@buildifier_prebuilt//:buildifier", starlark = "@buildifier_prebuilt//:buildifier",
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
# TODO(SERVER-101034): Enable rules_lint shfmt after sh files are reformatted with .editorconfig
# shell = "@shfmt//:shfmt",
) )

View File

@ -82,21 +82,6 @@ def run_rules_lint(
return True return True
def run_shellscripts_linters(shellscripts_linters: pathlib.Path, check: bool) -> bool:
try:
command = [str(shellscripts_linters)]
if not check:
print("Running shellscripts formatter")
command.append("fix")
else:
print("Running shellscripts linter")
repo_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
subprocess.run(command, check=True, env=os.environ, cwd=repo_path)
except subprocess.CalledProcessError:
return False
return True
def run_prettier( def run_prettier(
prettier: pathlib.Path, check: bool, files_to_format: Union[List[str], str] = "all" prettier: pathlib.Path, check: bool, files_to_format: Union[List[str], str] = "all"
) -> bool: ) -> bool:
@ -169,12 +154,6 @@ def main() -> int:
parser.add_argument( parser.add_argument(
"--prettier", help="Set the path to prettier", required=True, type=pathlib.Path "--prettier", help="Set the path to prettier", required=True, type=pathlib.Path
) )
parser.add_argument(
"--shellscripts-linters",
help="Set the path to shellscripts_linters",
required=True,
type=pathlib.Path,
)
parser.add_argument( parser.add_argument(
"--rules-lint-format", "--rules-lint-format",
help="Set the path to rules_lint's formatter", help="Set the path to rules_lint's formatter",
@ -200,7 +179,6 @@ def main() -> int:
args = parser.parse_args() args = parser.parse_args()
prettier_path: pathlib.Path = args.prettier.resolve() prettier_path: pathlib.Path = args.prettier.resolve()
shellscripts_linters_path: pathlib.Path = args.shellscripts_linters.resolve()
os.chdir(default_dir) os.chdir(default_dir)
@ -235,7 +213,6 @@ def main() -> int:
if run_rules_lint( if run_rules_lint(
args.rules_lint_format, args.rules_lint_format_check, args.check, files_to_format args.rules_lint_format, args.rules_lint_format_check, args.check, files_to_format
) )
and run_shellscripts_linters(shellscripts_linters_path, args.check)
and run_prettier(prettier_path, args.check, files_to_format) and run_prettier(prettier_path, args.check, files_to_format)
else 1 else 1
) )

View File

@ -234,7 +234,7 @@ generate_dockerfile() {
fi fi
mkdir -p "$output_dir" mkdir -p "$output_dir"
cat << EOF > "$output_dir/dockerfile" cat <<EOF >"$output_dir/dockerfile"
# DO NOT EDIT. # DO NOT EDIT.
# #
# This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin # This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin

View File

@ -111,18 +111,6 @@ py_binary(
], ],
) )
sh_binary(
name = "shellscripts_linters",
srcs = ["shellscripts-linters.sh"],
data = [
"@shfmt",
],
env = {
"SHFMT_PATH": "$(rootpath @shfmt//:shfmt)",
},
visibility = ["//visibility:public"],
)
py_library( py_library(
name = "mongo_toolchain", name = "mongo_toolchain",
srcs = [ srcs = [

View File

@ -24,8 +24,8 @@ if [[ -f "$HOME/.zshrc" ]]; then
rc_file="$HOME/.zshrc" rc_file="$HOME/.zshrc"
fi fi
if ! command -v db-contrib-tool &> /dev/null; then if ! command -v db-contrib-tool &>/dev/null; then
if ! python3 -c "import sys; sys.exit(sys.version_info < (3, 7))" &> /dev/null; then if ! python3 -c "import sys; sys.exit(sys.version_info < (3, 7))" &>/dev/null; then
actual_version=$(python3 -c 'import sys; print(sys.version)') actual_version=$(python3 -c 'import sys; print(sys.version)')
echo "You must have python3.7+ installed. Detected version $actual_version." echo "You must have python3.7+ installed. Detected version $actual_version."
echo "To avoid unexpected issues, python3.7+ will not be automatically installed." echo "To avoid unexpected issues, python3.7+ will not be automatically installed."
@ -38,12 +38,12 @@ if ! command -v db-contrib-tool &> /dev/null; then
exit 1 exit 1
fi fi
if command -v pipx &> /dev/null; then if command -v pipx &>/dev/null; then
echo "Found pipx: $(command -v pipx)." echo "Found pipx: $(command -v pipx)."
echo "Using it to install 'db-contrib-tool'." echo "Using it to install 'db-contrib-tool'."
echo echo
pipx ensurepath &> /dev/null pipx ensurepath &>/dev/null
if [[ -f "$rc_file" ]]; then if [[ -f "$rc_file" ]]; then
source "$rc_file" source "$rc_file"
fi fi
@ -51,7 +51,7 @@ if ! command -v db-contrib-tool &> /dev/null; then
pipx install db-contrib-tool --python $(command -v python3) --force pipx install db-contrib-tool --python $(command -v python3) --force
echo echo
else else
if ! python3 -m pipx --version &> /dev/null; then if ! python3 -m pipx --version &>/dev/null; then
echo "Couldn't find pipx. Installing it as python3 module:" echo "Couldn't find pipx. Installing it as python3 module:"
echo " $(command -v python3) -m pip install pipx" echo " $(command -v python3) -m pip install pipx"
echo echo
@ -64,7 +64,7 @@ if ! command -v db-contrib-tool &> /dev/null; then
echo echo
fi fi
python3 -m pipx ensurepath &> /dev/null python3 -m pipx ensurepath &>/dev/null
if [[ -f "$rc_file" ]]; then if [[ -f "$rc_file" ]]; then
source "$rc_file" source "$rc_file"
fi fi

View File

@ -26,43 +26,36 @@ mkdir -p "$repodir/yum/redhat"
# to support different $releasever values in yum repo configurations # to support different $releasever values in yum repo configurations
# #
if [ ! -e "$repodir/yum/redhat/7Server" ] if [ ! -e "$repodir/yum/redhat/7Server" ]; then
then
ln -s 7 "$repodir/yum/redhat/7Server" ln -s 7 "$repodir/yum/redhat/7Server"
fi fi
if [ ! -e "$repodir/yum/redhat/6Server" ] if [ ! -e "$repodir/yum/redhat/6Server" ]; then
then
ln -s 6 "$repodir/yum/redhat/6Server" ln -s 6 "$repodir/yum/redhat/6Server"
fi fi
if [ ! -e "$repodir/yum/redhat/5Server" ] if [ ! -e "$repodir/yum/redhat/5Server" ]; then
then
ln -s 5 "$repodir/yum/redhat/5Server" ln -s 5 "$repodir/yum/redhat/5Server"
fi fi
echo "Scanning and copying package files from $source_dir" echo "Scanning and copying package files from $source_dir"
echo ". = skipping existing file, @ = copying file" echo ". = skipping existing file, @ = copying file"
for package in $(find "$source_dir" -not \( -path "$repodir" -prune \) -and \( -name \*.rpm -o -name \*.deb -o -name Release \)) for package in $(find "$source_dir" -not \( -path "$repodir" -prune \) -and \( -name \*.rpm -o -name \*.deb -o -name Release \)); do
do
new_package_location="$repodir$(echo "$package" | sed 's/\/var\/www-enterprise\/[^\/]*//;')" new_package_location="$repodir$(echo "$package" | sed 's/\/var\/www-enterprise\/[^\/]*//;')"
# skip if the directory structure looks weird # skip if the directory structure looks weird
# #
if echo "$new_package_location" | grep -q /repo/ if echo "$new_package_location" | grep -q /repo/; then
then
continue continue
fi fi
# skip if not enterprise package # skip if not enterprise package
# #
if ! echo "$new_package_location" | grep -q enterprise if ! echo "$new_package_location" | grep -q enterprise; then
then
continue continue
fi fi
# skip if it's already there # skip if it's already there
# #
if [ -e "$new_package_location" -a "$(basename "$package")" != "Release" ] if [ -e "$new_package_location" -a "$(basename "$package")" != "Release" ]; then
then
echo -n . echo -n .
else else
mkdir -p "$(dirname "$new_package_location")" mkdir -p "$(dirname "$new_package_location")"
@ -74,33 +67,28 @@ echo
# packages are in place, now create metadata # packages are in place, now create metadata
# #
for debian_dir in "$repodir"/apt/ubuntu "$repodir"/apt/debian for debian_dir in "$repodir"/apt/ubuntu "$repodir"/apt/debian; do
do
cd "$debian_dir" cd "$debian_dir"
for section_dir in $(find dists -type d -name multiverse -o -name main) for section_dir in $(find dists -type d -name multiverse -o -name main); do
do for arch_dir in "$section_dir"/{binary-i386,binary-amd64}; do
for arch_dir in "$section_dir"/{binary-i386,binary-amd64}
do
echo "Generating Packages file under $debian_dir/$arch_dir" echo "Generating Packages file under $debian_dir/$arch_dir"
if [ ! -d $arch_dir ] if [ ! -d $arch_dir ]; then
then
mkdir $arch_dir mkdir $arch_dir
fi fi
dpkg-scanpackages --multiversion "$arch_dir" > "$arch_dir"/Packages dpkg-scanpackages --multiversion "$arch_dir" >"$arch_dir"/Packages
gzip -9c "$arch_dir"/Packages > "$arch_dir"/Packages.gz gzip -9c "$arch_dir"/Packages >"$arch_dir"/Packages.gz
done done
done done
for release_file in $(find "$debian_dir" -name Release) for release_file in $(find "$debian_dir" -name Release); do
do
release_dir=$(dirname "$release_file") release_dir=$(dirname "$release_file")
echo "Generating Release file under $release_dir" echo "Generating Release file under $release_dir"
cd $release_dir cd $release_dir
tempfile=$(mktemp /tmp/ReleaseXXXXXX) tempfile=$(mktemp /tmp/ReleaseXXXXXX)
tempfile2=$(mktemp /tmp/ReleaseXXXXXX) tempfile2=$(mktemp /tmp/ReleaseXXXXXX)
mv Release $tempfile mv Release $tempfile
head -7 $tempfile > $tempfile2 head -7 $tempfile >$tempfile2
apt-ftparchive release . >> $tempfile2 apt-ftparchive release . >>$tempfile2
cp $tempfile2 Release cp $tempfile2 Release
chmod 644 Release chmod 644 Release
rm Release.gpg rm Release.gpg
@ -118,28 +106,23 @@ done
# /var/www-enterprise/repo.consolidated/apt/ubuntu/dists/precise/mongodb-enterprise/unstable -> 2.5 # /var/www-enterprise/repo.consolidated/apt/ubuntu/dists/precise/mongodb-enterprise/unstable -> 2.5
# /var/www-enterprise/repo.consolidated/apt/debian/dists/wheezy/mongodb-enterprise/unstable -> 2.5 # /var/www-enterprise/repo.consolidated/apt/debian/dists/wheezy/mongodb-enterprise/unstable -> 2.5
# #
for unstable_branch_dir in "$repodir"/yum/redhat/*/*/$unstable_branch "$repodir"/yum/amazon/*/*/$unstable_branch "$repodir"/apt/debian/dists/*/*/$unstable_branch "$repodir"/apt/ubuntu/dists/*/*/$unstable_branch "$repodir"/zypper/suse/*/*/$unstable_branch for unstable_branch_dir in "$repodir"/yum/redhat/*/*/$unstable_branch "$repodir"/yum/amazon/*/*/$unstable_branch "$repodir"/apt/debian/dists/*/*/$unstable_branch "$repodir"/apt/ubuntu/dists/*/*/$unstable_branch "$repodir"/zypper/suse/*/*/$unstable_branch; do
do
full_unstable_path=$(dirname "$unstable_branch_dir")/unstable full_unstable_path=$(dirname "$unstable_branch_dir")/unstable
if [ -e "$unstable_branch_dir" -a ! -e "$full_unstable_path" ] if [ -e "$unstable_branch_dir" -a ! -e "$full_unstable_path" ]; then
then
echo "Linking unstable branch directory $unstable_branch_dir to $full_unstable_path" echo "Linking unstable branch directory $unstable_branch_dir to $full_unstable_path"
ln -s $unstable_branch $full_unstable_path ln -s $unstable_branch $full_unstable_path
fi fi
done done
for stable_branch_dir in "$repodir"/yum/redhat/*/*/$stable_branch "$repodir"/yum/amazon/*/*/$stable_branch "$repodir"/apt/debian/dists/*/*/$stable_branch "$repodir"/apt/ubuntu/dists/*/*/$stable_branch "$repodir"/zypper/suse/*/*/$stable_branch for stable_branch_dir in "$repodir"/yum/redhat/*/*/$stable_branch "$repodir"/yum/amazon/*/*/$stable_branch "$repodir"/apt/debian/dists/*/*/$stable_branch "$repodir"/apt/ubuntu/dists/*/*/$stable_branch "$repodir"/zypper/suse/*/*/$stable_branch; do
do
full_stable_path=$(dirname "$stable_branch_dir")/stable full_stable_path=$(dirname "$stable_branch_dir")/stable
if [ -e "$stable_branch_dir" -a ! -e "$full_stable_path" ] if [ -e "$stable_branch_dir" -a ! -e "$full_stable_path" ]; then
then
echo "Linking stable branch directory $stable_branch_dir to $full_stable_path" echo "Linking stable branch directory $stable_branch_dir to $full_stable_path"
ln -s $stable_branch $full_stable_path ln -s $stable_branch $full_stable_path
fi fi
done done
for rpm_dir in $(find "$repodir"/yum/redhat "$repodir"/zypper/suse -type d -name x86_64 -o -name i386) for rpm_dir in $(find "$repodir"/yum/redhat "$repodir"/zypper/suse -type d -name x86_64 -o -name i386); do
do
echo "Generating redhat repo metadata under $rpm_dir" echo "Generating redhat repo metadata under $rpm_dir"
cd "$rpm_dir" cd "$rpm_dir"
createrepo . createrepo .

View File

@ -26,43 +26,36 @@ mkdir -p "$repodir/yum/redhat"
# to support different $releasever values in yum repo configurations # to support different $releasever values in yum repo configurations
# #
if [ ! -e "$repodir/yum/redhat/6Server" ] if [ ! -e "$repodir/yum/redhat/6Server" ]; then
then
ln -s 6 "$repodir/yum/redhat/6Server" ln -s 6 "$repodir/yum/redhat/6Server"
fi fi
if [ ! -e "$repodir/yum/redhat/7Server" ] if [ ! -e "$repodir/yum/redhat/7Server" ]; then
then
ln -s 7 "$repodir/yum/redhat/7Server" ln -s 7 "$repodir/yum/redhat/7Server"
fi fi
if [ ! -e "$repodir/yum/redhat/5Server" ] if [ ! -e "$repodir/yum/redhat/5Server" ]; then
then
ln -s 5 "$repodir/yum/redhat/5Server" ln -s 5 "$repodir/yum/redhat/5Server"
fi fi
echo "Scanning and copying package files from $source_dir" echo "Scanning and copying package files from $source_dir"
echo ". = skipping existing file, @ = copying file" echo ". = skipping existing file, @ = copying file"
for package in $(find "$source_dir" -not \( -path "$repodir" -prune \) -and \( -name \*.rpm -o -name \*.deb -o -name Release \)) for package in $(find "$source_dir" -not \( -path "$repodir" -prune \) -and \( -name \*.rpm -o -name \*.deb -o -name Release \)); do
do
new_package_location="$repodir$(echo "$package" | sed 's/\/var\/www-org\/[^\/]*//;')" new_package_location="$repodir$(echo "$package" | sed 's/\/var\/www-org\/[^\/]*//;')"
# skip if the directory structure looks weird # skip if the directory structure looks weird
# #
if echo "$new_package_location" | grep -q /repo/ if echo "$new_package_location" | grep -q /repo/; then
then
continue continue
fi fi
# skip if not community package # skip if not community package
# #
if ! echo "$new_package_location" | grep -q org if ! echo "$new_package_location" | grep -q org; then
then
continue continue
fi fi
# skip if it's already there # skip if it's already there
# #
if [ -e "$new_package_location" -a "$(basename "$package")" != "Release" ] if [ -e "$new_package_location" -a "$(basename "$package")" != "Release" ]; then
then
echo -n . echo -n .
else else
mkdir -p "$(dirname "$new_package_location")" mkdir -p "$(dirname "$new_package_location")"
@ -74,33 +67,28 @@ echo
# packages are in place, now create metadata # packages are in place, now create metadata
# #
for debian_dir in "$repodir"/apt/ubuntu "$repodir"/apt/debian for debian_dir in "$repodir"/apt/ubuntu "$repodir"/apt/debian; do
do
cd "$debian_dir" cd "$debian_dir"
for section_dir in $(find dists -type d -name multiverse -o -name main) for section_dir in $(find dists -type d -name multiverse -o -name main); do
do for arch_dir in "$section_dir"/{binary-i386,binary-amd64}; do
for arch_dir in "$section_dir"/{binary-i386,binary-amd64}
do
echo "Generating Packages file under $debian_dir/$arch_dir" echo "Generating Packages file under $debian_dir/$arch_dir"
if [ ! -d $arch_dir ] if [ ! -d $arch_dir ]; then
then
mkdir $arch_dir mkdir $arch_dir
fi fi
dpkg-scanpackages --multiversion "$arch_dir" > "$arch_dir"/Packages dpkg-scanpackages --multiversion "$arch_dir" >"$arch_dir"/Packages
gzip -9c "$arch_dir"/Packages > "$arch_dir"/Packages.gz gzip -9c "$arch_dir"/Packages >"$arch_dir"/Packages.gz
done done
done done
for release_file in $(find "$debian_dir" -name Release) for release_file in $(find "$debian_dir" -name Release); do
do
release_dir=$(dirname "$release_file") release_dir=$(dirname "$release_file")
echo "Generating Release file under $release_dir" echo "Generating Release file under $release_dir"
cd $release_dir cd $release_dir
tempfile=$(mktemp /tmp/ReleaseXXXXXX) tempfile=$(mktemp /tmp/ReleaseXXXXXX)
tempfile2=$(mktemp /tmp/ReleaseXXXXXX) tempfile2=$(mktemp /tmp/ReleaseXXXXXX)
mv Release $tempfile mv Release $tempfile
head -7 $tempfile > $tempfile2 head -7 $tempfile >$tempfile2
apt-ftparchive release . >> $tempfile2 apt-ftparchive release . >>$tempfile2
cp $tempfile2 Release cp $tempfile2 Release
chmod 644 Release chmod 644 Release
rm Release.gpg rm Release.gpg
@ -118,28 +106,23 @@ done
# /var/www-org/repo.consolidated/apt/ubuntu/dists/precise/mongodb-org/unstable -> 2.5 # /var/www-org/repo.consolidated/apt/ubuntu/dists/precise/mongodb-org/unstable -> 2.5
# /var/www-org/repo.consolidated/apt/debian/dists/wheezy/mongodb-org/unstable -> 2.5 # /var/www-org/repo.consolidated/apt/debian/dists/wheezy/mongodb-org/unstable -> 2.5
# #
for unstable_branch_dir in "$repodir"/yum/redhat/*/*/$unstable_branch "$repodir"/yum/amazon/*/*/$unstable_branch "$repodir"/apt/debian/dists/*/*/$unstable_branch "$repodir"/apt/ubuntu/dists/*/*/$unstable_branch "$repodir"/zypper/suse/*/*/$unstable_branch for unstable_branch_dir in "$repodir"/yum/redhat/*/*/$unstable_branch "$repodir"/yum/amazon/*/*/$unstable_branch "$repodir"/apt/debian/dists/*/*/$unstable_branch "$repodir"/apt/ubuntu/dists/*/*/$unstable_branch "$repodir"/zypper/suse/*/*/$unstable_branch; do
do
full_unstable_path=$(dirname "$unstable_branch_dir")/unstable full_unstable_path=$(dirname "$unstable_branch_dir")/unstable
if [ -e "$unstable_branch_dir" -a ! -e "$full_unstable_path" ] if [ -e "$unstable_branch_dir" -a ! -e "$full_unstable_path" ]; then
then
echo "Linking unstable branch directory $unstable_branch_dir to $full_unstable_path" echo "Linking unstable branch directory $unstable_branch_dir to $full_unstable_path"
ln -s $unstable_branch $full_unstable_path ln -s $unstable_branch $full_unstable_path
fi fi
done done
for stable_branch_dir in "$repodir"/yum/redhat/*/*/$stable_branch "$repodir"/yum/amazon/*/*/$stable_branch "$repodir"/apt/debian/dists/*/*/$stable_branch "$repodir"/apt/ubuntu/dists/*/*/$stable_branch "$repodir"/zypper/suse/*/*/$stable_branch for stable_branch_dir in "$repodir"/yum/redhat/*/*/$stable_branch "$repodir"/yum/amazon/*/*/$stable_branch "$repodir"/apt/debian/dists/*/*/$stable_branch "$repodir"/apt/ubuntu/dists/*/*/$stable_branch "$repodir"/zypper/suse/*/*/$stable_branch; do
do
full_stable_path=$(dirname "$stable_branch_dir")/stable full_stable_path=$(dirname "$stable_branch_dir")/stable
if [ -e "$stable_branch_dir" -a ! -e "$full_stable_path" ] if [ -e "$stable_branch_dir" -a ! -e "$full_stable_path" ]; then
then
echo "Linking stable branch directory $stable_branch_dir to $full_stable_path" echo "Linking stable branch directory $stable_branch_dir to $full_stable_path"
ln -s $stable_branch $full_stable_path ln -s $stable_branch $full_stable_path
fi fi
done done
for rpm_dir in $(find "$repodir"/yum/redhat "$repodir"/yum/amazon "$repodir"/zypper/suse -type d -name x86_64 -o -name i386) for rpm_dir in $(find "$repodir"/yum/redhat "$repodir"/yum/amazon "$repodir"/zypper/suse -type d -name x86_64 -o -name i386); do
do
echo "Generating redhat repo metadata under $rpm_dir" echo "Generating redhat repo metadata under $rpm_dir"
cd "$rpm_dir" cd "$rpm_dir"
createrepo . createrepo .

View File

@ -12,7 +12,7 @@ user_group=$USER:$(id -Gn $USER | cut -f1 -d ' ')
# _usage_: Provides usage infomation # _usage_: Provides usage infomation
function _usage_ { function _usage_ {
cat << EOF cat <<EOF
usage: $0 options usage: $0 options
This script supports the following parameters for Windows & Linux platforms: This script supports the following parameters for Windows & Linux platforms:
-d <deviceNames>, REQUIRED, Space separated list of devices to mount /data on, -d <deviceNames>, REQUIRED, Space separated list of devices to mount /data on,
@ -31,10 +31,8 @@ This script supports the following parameters for Windows & Linux platforms:
EOF EOF
} }
# Parse command line options # Parse command line options
while getopts "d:l:o:r:t:u:?" option while getopts "d:l:o:r:t:u:?" option; do
do
case $option in case $option in
d) d)
data_device_names=$OPTARG data_device_names=$OPTARG
@ -54,7 +52,7 @@ do
u) u)
user_group=$OPTARG user_group=$OPTARG
;; ;;
\?|*) \? | *)
_usage_ _usage_
exit 0 exit 0
;; ;;
@ -72,8 +70,7 @@ function mount_drive {
# Determine how many devices were specified # Determine how many devices were specified
local num_devices=0 local num_devices=0
for device_name in $device_names for device_name in $device_names; do
do
local devices="$devices /dev/$device_name" local devices="$devices /dev/$device_name"
let num_devices=num_devices+1 let num_devices=num_devices+1
done done
@ -93,8 +90,7 @@ function mount_drive {
local drive=$device_names local drive=$device_names
local system_drive=c local system_drive=c
while true; while true; do
do
sleep $drive_poll_delay sleep $drive_poll_delay
echo "Looking for drive '$drive' to mount $root_dir" echo "Looking for drive '$drive' to mount $root_dir"
if [ -d /cygdrive/$drive ]; then if [ -d /cygdrive/$drive ]; then
@ -105,8 +101,7 @@ function mount_drive {
cmd.exe /c mklink /J $system_drive:\\$root_dir $drive:\\$root_dir cmd.exe /c mklink /J $system_drive:\\$root_dir $drive:\\$root_dir
ln -s /cygdrive/$drive/$root_dir /$root_dir ln -s /cygdrive/$drive/$root_dir /$root_dir
setfacl -s user::rwx,group::rwx,other::rwx /cygdrive/$drive/$root_dir setfacl -s user::rwx,group::rwx,other::rwx /cygdrive/$drive/$root_dir
for sub_dir in $sub_dirs for sub_dir in $sub_dirs; do
do
mkdir -p /cygdrive/$drive/$root_dir/$sub_dir mkdir -p /cygdrive/$drive/$root_dir/$sub_dir
done done
chown -R $user_group /cygdrive/$system_drive/$root_dir chown -R $user_group /cygdrive/$system_drive/$root_dir
@ -144,7 +139,7 @@ function mount_drive {
/sbin/udevadm control --stop-exec-queue /sbin/udevadm control --stop-exec-queue
yes | /sbin/mdadm --create $device_name --level=0 -c256 --raid-devices=$num_devices $devices yes | /sbin/mdadm --create $device_name --level=0 -c256 --raid-devices=$num_devices $devices
/sbin/udevadm control --start-exec-queue /sbin/udevadm control --start-exec-queue
/sbin/mdadm --detail --scan > /etc/mdadm.conf /sbin/mdadm --detail --scan >/etc/mdadm.conf
/sbin/blockdev --setra 32 $device_name /sbin/blockdev --setra 32 $device_name
else else
device_name="/dev/$device_names" device_name="/dev/$device_names"
@ -165,8 +160,7 @@ function mount_drive {
mkdir /$root_dir || true mkdir /$root_dir || true
chmod 777 /$root_dir chmod 777 /$root_dir
mount -t $fs_type "UUID=$device_uuid" /$root_dir mount -t $fs_type "UUID=$device_uuid" /$root_dir
for sub_dir in $sub_dirs for sub_dir in $sub_dirs; do
do
mkdir -p /$root_dir/$sub_dir mkdir -p /$root_dir/$sub_dir
chmod 1777 /$root_dir/$sub_dir chmod 1777 /$root_dir/$sub_dir
done done

View File

@ -40,7 +40,7 @@ while getopts p:fin opt; do
esac esac
done done
run () { run() {
echo "$@" echo "$@"
if [[ "${dry_run}" == 1 ]]; then if [[ "${dry_run}" == 1 ]]; then
return return
@ -86,7 +86,7 @@ if ! "${py3}" -m pip show poetry &>/dev/null; then
fi fi
# we'll need to use pip this time around # we'll need to use pip this time around
if (( need_poetry_install )); then if ((need_poetry_install)); then
run "${py3}" -m pip install "${pip_opts[@]}" -r poetry_requirements.txt run "${py3}" -m pip install "${pip_opts[@]}" -r poetry_requirements.txt
fi fi

View File

@ -32,6 +32,6 @@ else
mv engflow.key $HOME/.engflow/creds mv engflow.key $HOME/.engflow/creds
chown $USER $HOME/.engflow/creds/engflow.crt $HOME/.engflow/creds/engflow.key chown $USER $HOME/.engflow/creds/engflow.crt $HOME/.engflow/creds/engflow.key
chmod 600 $HOME/.engflow/creds/engflow.crt $HOME/.engflow/creds/engflow.key chmod 600 $HOME/.engflow/creds/engflow.crt $HOME/.engflow/creds/engflow.key
echo "common --tls_client_certificate=$HOME/.engflow/creds/engflow.crt" >> $HOME/.bazelrc echo "common --tls_client_certificate=$HOME/.engflow/creds/engflow.crt" >>$HOME/.bazelrc
echo "common --tls_client_key=$HOME/.engflow/creds/engflow.key" >> $HOME/.bazelrc echo "common --tls_client_key=$HOME/.engflow/creds/engflow.key" >>$HOME/.bazelrc
fi fi

View File

@ -1,41 +0,0 @@
#!/bin/bash
set +o errexit
shfmt=shfmt
if [ -n "$SHFMT_PATH" ]; then
shfmt=$(readlink $SHFMT_PATH)
fi
if [ -n "$BUILD_WORKSPACE_DIRECTORY" ]; then
cd $BUILD_WORKSPACE_DIRECTORY
fi
if ! command -v $shfmt &>/dev/null; then
echo "Could not find shfmt at $shfmt"
exit 1
fi
lint_dirs="evergreen"
if [ "$1" = "fix" ]; then
$shfmt -w -i 2 -bn -sr "$lint_dirs"
fi
output_file="shfmt_output.txt"
exit_code=0
$shfmt -d -i 2 -bn -sr "$lint_dirs" >"$output_file"
if [ -s "$output_file" ]; then
echo "ERROR: Found formatting errors in shell script files in directories: $lint_dirs"
echo ""
cat "$output_file"
echo ""
echo "To fix formatting errors run"
echo ""
echo " ./buildscripts/shellscripts-linters.sh fix"
echo ""
exit_code=1
fi
rm -rf "$output_file"
exit "$exit_code"

View File

@ -8,4 +8,3 @@ set -vx
NAME=protobuf NAME=protobuf
VERSION="mongo/v4.25.0" VERSION="mongo/v4.25.0"

View File

@ -5,14 +5,14 @@ cd "$BASEDIR/../"
yamllint -c etc/yamllint_config.yml buildscripts etc jstests yamllint -c etc/yamllint_config.yml buildscripts etc jstests
PATH="$PATH:$HOME" evergreen evaluate etc/evergreen.yml > etc/evaluated_evergreen.yml PATH="$PATH:$HOME" evergreen evaluate etc/evergreen.yml >etc/evaluated_evergreen.yml
PATH="$PATH:$HOME" evergreen evaluate etc/evergreen_nightly.yml > etc/evaluated_evergreen_nightly.yml PATH="$PATH:$HOME" evergreen evaluate etc/evergreen_nightly.yml >etc/evaluated_evergreen_nightly.yml
# Remove references to the DSI repo before evergreen evaluate. # Remove references to the DSI repo before evergreen evaluate.
# The DSI module references break 'evaluate', the system_perf config should # The DSI module references break 'evaluate', the system_perf config should
# parse without them, and we don't want changes to the DSI repository to # parse without them, and we don't want changes to the DSI repository to
# break checking that the rest of the imports etc. work. # break checking that the rest of the imports etc. work.
awk '/lint_yaml trim start/{drop=1} /lint_yaml trim end/{drop=0} !drop' etc/system_perf.yml > etc/trimmed_system_perf.yml awk '/lint_yaml trim start/{drop=1} /lint_yaml trim end/{drop=0} !drop' etc/system_perf.yml >etc/trimmed_system_perf.yml
PATH="$PATH:$HOME" evergreen evaluate etc/trimmed_system_perf.yml > etc/evaluated_system_perf.yml PATH="$PATH:$HOME" evergreen evaluate etc/trimmed_system_perf.yml >etc/evaluated_system_perf.yml
python -m evergreen_lint -c ./etc/evergreen_lint.yml lint python -m evergreen_lint -c ./etc/evergreen_lint.yml lint

View File

@ -1,5 +1,5 @@
silent_grep() { silent_grep() {
command grep -q > /dev/null 2>&1 "$@" command grep -q "$@" >/dev/null 2>&1
} }
idem_file_append() { idem_file_append() {
@ -19,10 +19,10 @@ idem_file_append() {
local end_marker="# END $2" local end_marker="# END $2"
if ! silent_grep "^$start_marker" "$1"; then if ! silent_grep "^$start_marker" "$1"; then
{ {
echo -e "\n$start_marker"; echo -e "\n$start_marker"
echo -e "$3"; echo -e "$3"
echo -e "$end_marker"; echo -e "$end_marker"
} >> "$1" } >>"$1"
fi fi
} }
@ -30,7 +30,8 @@ setup_bash() {
# Bash profile should source .bashrc # Bash profile should source .bashrc
echo "################################################################################" echo "################################################################################"
echo "Setting up bash..." echo "Setting up bash..."
local block=$(cat <<BLOCK local block=$(
cat <<BLOCK
if [[ -f ~/.bashrc ]]; then if [[ -f ~/.bashrc ]]; then
source ~/.bashrc source ~/.bashrc
fi fi
@ -67,7 +68,7 @@ setup_poetry() {
echo "################################################################################" echo "################################################################################"
echo "Installing 'poetry' command..." echo "Installing 'poetry' command..."
export PATH="$PATH:$HOME/.local/bin" export PATH="$PATH:$HOME/.local/bin"
if command -v poetry &> /dev/null; then if command -v poetry &>/dev/null; then
echo "'poetry' command exists; skipping setup" echo "'poetry' command exists; skipping setup"
else else
pipx install poetry --pip-args="-r $(pwd)/poetry_requirements.txt" pipx install poetry --pip-args="-r $(pwd)/poetry_requirements.txt"
@ -78,7 +79,7 @@ setup_poetry() {
setup_pipx() { setup_pipx() {
echo "################################################################################" echo "################################################################################"
echo "Installing 'pipx' command..." echo "Installing 'pipx' command..."
if command -v pipx &> /dev/null; then if command -v pipx &>/dev/null; then
echo "'pipx' command exists; skipping setup" echo "'pipx' command exists; skipping setup"
else else
export PATH="$PATH:$HOME/.local/bin" export PATH="$PATH:$HOME/.local/bin"
@ -112,7 +113,7 @@ setup_db_contrib_tool() {
echo "################################################################################" echo "################################################################################"
echo "Installing 'db-contrib-tool' command..." echo "Installing 'db-contrib-tool' command..."
export PATH="$PATH:$HOME/.local/bin" export PATH="$PATH:$HOME/.local/bin"
if command -v db-contrib-tool &> /dev/null; then if command -v db-contrib-tool &>/dev/null; then
echo "'db-contrib-tool' command exists; skipping setup" echo "'db-contrib-tool' command exists; skipping setup"
else else
pipx install db-contrib-tool pipx install db-contrib-tool

View File

@ -296,11 +296,6 @@ sh_binary(
srcs = ["lint_fuzzer_sanity_patch.sh"], srcs = ["lint_fuzzer_sanity_patch.sh"],
) )
sh_binary(
name = "lint_shellscripts",
srcs = ["lint_shellscripts.sh"],
)
sh_binary( sh_binary(
name = "lint_yaml", name = "lint_yaml",
srcs = ["lint_yaml.sh"], srcs = ["lint_yaml.sh"],

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
set -o errexit set -o errexit
@ -43,7 +43,7 @@ echo "docker daemon.json: set data-root to /data/mci/docker"
sudo service docker start sudo service docker start
# Login # Login
echo "${antithesis_repo_key}" > mongodb.key.json echo "${antithesis_repo_key}" >mongodb.key.json
cat mongodb.key.json | sudo docker login -u _json_key https://us-central1-docker.pkg.dev --password-stdin cat mongodb.key.json | sudo docker login -u _json_key https://us-central1-docker.pkg.dev --password-stdin
rm mongodb.key.json rm mongodb.key.json
@ -64,7 +64,7 @@ timeout -v 1800 docker exec workload buildscripts/resmoke.py run --suite ${suite
RET=$? RET=$?
set -o errexit set -o errexit
docker-compose -f docker_compose/${suite}/docker-compose.yml logs > docker_logs.txt docker-compose -f docker_compose/${suite}/docker-compose.yml logs >docker_logs.txt
docker-compose -f docker_compose/${suite}/docker-compose.yml down docker-compose -f docker_compose/${suite}/docker-compose.yml down
# Change the permissions of all of the files in the docker compose directory to the current user. # Change the permissions of all of the files in the docker compose directory to the current user.

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
source "$DIR/bazel_utility_functions.sh" source "$DIR/bazel_utility_functions.sh"
( (

View File

@ -8,7 +8,7 @@
# * ${args} - List of additional Bazel arguments (e.g.: "--config=clang-tidy") # * ${args} - List of additional Bazel arguments (e.g.: "--config=clang-tidy")
# Needed for evergreen scripts that use evergreen expansions and utility methods. # Needed for evergreen scripts that use evergreen expansions and utility methods.
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -146,8 +146,8 @@ fi
for i in {1..3}; do for i in {1..3}; do
eval ${TIMEOUT_CMD} $BAZEL_BINARY build --verbose_failures $LOCAL_ARG ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \ eval ${TIMEOUT_CMD} $BAZEL_BINARY build --verbose_failures $LOCAL_ARG ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \
--define=MONGO_VERSION=${version} ${patch_compile_flags} ${targets} 2>&1 | tee bazel_stdout.log \ --define=MONGO_VERSION=${version} ${patch_compile_flags} ${targets} 2>&1 | tee bazel_stdout.log &&
&& RET=0 && break || RET=$? && sleep 60 RET=0 && break || RET=$? && sleep 60
if [ $RET -eq 124 ]; then if [ $RET -eq 124 ]; then
echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..." echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..."
else else

View File

@ -8,7 +8,7 @@
# * ${args} - Extra command line args to pass to "bazel coverage" # * ${args} - Extra command line args to pass to "bazel coverage"
# Needed for evergreen scripts that use evergreen expansions and utility methods. # Needed for evergreen scripts that use evergreen expansions and utility methods.
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -24,7 +24,7 @@ eval echo "Execution environment: Args: ${args} Target: ${target}"
BAZEL_BINARY=bazel BAZEL_BINARY=bazel
# Print command being run to file that can be uploaded # Print command being run to file that can be uploaded
echo "python buildscripts/install_bazel.py" > bazel-invocation.txt echo "python buildscripts/install_bazel.py" >bazel-invocation.txt
echo " bazel coverage ${args} ${target}" >> bazel-invocation.txt echo " bazel coverage ${args} ${target}" >>bazel-invocation.txt
$BAZEL_BINARY coverage ${args} ${target} $BAZEL_BINARY coverage ${args} ${target}

View File

@ -8,7 +8,7 @@
# * ${redact_args} - If set, redact the args in the report # * ${redact_args} - If set, redact the args in the report
# Needed for evergreen scripts that use evergreen expansions and utility methods. # Needed for evergreen scripts that use evergreen expansions and utility methods.
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -38,7 +38,7 @@ BAZEL_BINARY=$(bazel_get_binary_path)
# AL2 stores certs in a nonstandard location # AL2 stores certs in a nonstandard location
if [[ -f /etc/os-release ]]; then if [[ -f /etc/os-release ]]; then
DISTRO=$(awk -F '[="]*' '/^PRETTY_NAME/ { print $2 }' < /etc/os-release) DISTRO=$(awk -F '[="]*' '/^PRETTY_NAME/ { print $2 }' </etc/os-release)
if [[ $DISTRO == "Amazon Linux 2" ]]; then if [[ $DISTRO == "Amazon Linux 2" ]]; then
export SSL_CERT_DIR=/etc/pki/tls/certs export SSL_CERT_DIR=/etc/pki/tls/certs
export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
@ -55,14 +55,14 @@ else
fi fi
# Print command being run to file that can be uploaded # Print command being run to file that can be uploaded
echo "python buildscripts/install_bazel.py" > bazel-invocation.txt echo "python buildscripts/install_bazel.py" >bazel-invocation.txt
echo "bazel run --verbose_failures ${bazel_compile_flags} ${task_compile_flags} ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >> bazel-invocation.txt echo "bazel run --verbose_failures ${bazel_compile_flags} ${task_compile_flags} ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >>bazel-invocation.txt
# Run bazel command, retrying up to five times # Run bazel command, retrying up to five times
MAX_ATTEMPTS=5 MAX_ATTEMPTS=5
for ((i = 1; i <= $MAX_ATTEMPTS; i++)); do for ((i = 1; i <= $MAX_ATTEMPTS; i++)); do
eval $env $BAZEL_BINARY run --verbose_failures $LOCAL_ARG ${target} ${args} >> bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 10 eval $env $BAZEL_BINARY run --verbose_failures $LOCAL_ARG ${target} ${args} >>bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 10
if [ $i -lt $MAX_ATTEMPTS ]; then echo "Bazel failed to execute, retrying ($(($i + 1)) of $MAX_ATTEMPTS attempts)... " >> bazel_output.log 2>&1; fi if [ $i -lt $MAX_ATTEMPTS ]; then echo "Bazel failed to execute, retrying ($(($i + 1)) of $MAX_ATTEMPTS attempts)... " >>bazel_output.log 2>&1; fi
$BAZEL_BINARY shutdown $BAZEL_BINARY shutdown
done done

View File

@ -5,7 +5,7 @@
# * ${targets} - Test targets # * ${targets} - Test targets
# * ${bazel_args} - Extra command line args to pass to "bazel test" # * ${bazel_args} - Extra command line args to pass to "bazel test"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -64,7 +64,7 @@ if [ -n "${test_timeout_sec}" ]; then
fi fi
ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} --define=MONGO_VERSION=${version} ${patch_compile_flags}" ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} --define=MONGO_VERSION=${version} ${patch_compile_flags}"
echo ${ALL_FLAGS} > .bazel_build_flags echo ${ALL_FLAGS} >.bazel_build_flags
set +o errexit set +o errexit
@ -80,8 +80,8 @@ for i in {1..3}; do
done done
for i in {1..3}; do for i in {1..3}; do
eval ${TIMEOUT_CMD} ${BAZEL_BINARY} test ${ALL_FLAGS} ${targets} 2>&1 | tee bazel_stdout.log \ eval ${TIMEOUT_CMD} ${BAZEL_BINARY} test ${ALL_FLAGS} ${targets} 2>&1 | tee bazel_stdout.log &&
&& RET=0 && break || RET=$? && sleep 1 RET=0 && break || RET=$? && sleep 1
if [ $RET -eq 124 ]; then if [ $RET -eq 124 ]; then
echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..." echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..."
else else

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -2,21 +2,21 @@ failed_setup=false
source ~/.bashrc source ~/.bashrc
if command -v pipx &> /dev/null; then if command -v pipx &>/dev/null; then
echo "'pipx' command exists" echo "'pipx' command exists"
else else
echo "pipx command not found - failed setup" echo "pipx command not found - failed setup"
failed_setup=true failed_setup=true
fi fi
if command -v poetry &> /dev/null; then if command -v poetry &>/dev/null; then
echo "'poetry' command exists" echo "'poetry' command exists"
else else
echo "poetry command not found - failed setup" echo "poetry command not found - failed setup"
failed_setup=true failed_setup=true
fi fi
if command -v db-contrib-tool &> /dev/null; then if command -v db-contrib-tool &>/dev/null; then
echo "'db-contrib-tool' command exists" echo "'db-contrib-tool' command exists"
else else
echo "db-contrib-tool command not found - failed setup" echo "db-contrib-tool command not found - failed setup"
@ -26,7 +26,7 @@ fi
if test -d "./python3-venv"; then if test -d "./python3-venv"; then
echo "Venv directory exists, checking activation" echo "Venv directory exists, checking activation"
. python3-venv/bin/activate . python3-venv/bin/activate
./buildscripts/resmoke.py run --help &> /dev/null ./buildscripts/resmoke.py run --help &>/dev/null
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
echo "Virtual workstation set up correctly" echo "Virtual workstation set up correctly"
else else

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -10,5 +10,5 @@ OUTPUT_FILE="build/benchmarks.txt"
# Concatenate all text files in the directory into the output file # Concatenate all text files in the directory into the output file
for file in build/*_bm.txt; do for file in build/*_bm.txt; do
cat "$file" >> "$OUTPUT_FILE" cat "$file" >>"$OUTPUT_FILE"
done done

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -10,7 +10,7 @@ python buildscripts/install_bazel.py
bazel_bin="$HOME/.local/bin/bazelisk" bazel_bin="$HOME/.local/bin/bazelisk"
# number of parallel jobs to use for build. # number of parallel jobs to use for build.
# Even with scale=0 (the default), bc command adds decimal digits in case of multiplication. Division by 1 gives us a whole number with scale=0 # Even with scale=0 (the default), bc command adds decimal digits in case of multiplication. Division by 1 gives us a whole number with scale=0
bazel_jobs=$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) * .85 / 1") bazel_jobs=$(bc <<<"$(grep -c '^processor' /proc/cpuinfo) * .85 / 1")
build_config="--config=local --jobs=$bazel_jobs --compiler_type=gcc --opt=off --dbg=False --allocator=system" build_config="--config=local --jobs=$bazel_jobs --compiler_type=gcc --opt=off --dbg=False --allocator=system"
bazel_query='mnemonic("CppCompile|LinkCompile", filter(//src/mongo, deps(//:install-core)) except //src/mongo/db/modules/enterprise/src/streams/third_party/...)' bazel_query='mnemonic("CppCompile|LinkCompile", filter(//src/mongo, deps(//:install-core)) except //src/mongo/db/modules/enterprise/src/streams/third_party/...)'
bazel_cache="--output_user_root=$workdir/bazel_cache" bazel_cache="--output_user_root=$workdir/bazel_cache"

View File

@ -3,7 +3,7 @@
# This script verifies that specific symbols, and specific symbols only are # This script verifies that specific symbols, and specific symbols only are
# exported in mongo_crypt_v1.so # exported in mongo_crypt_v1.so
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
cd src cd src

View File

@ -1,8 +1,8 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
cd src cd src
set -o errexit set -o errexit
activate_venv activate_venv
$python -c 'import socket; num_nodes = 5; print("\n".join(["%s:%d" % (socket.gethostname(), port) for port in range(20000, 20000 + num_nodes)]))' > nodes.txt $python -c 'import socket; num_nodes = 5; print("\n".join(["%s:%d" % (socket.gethostname(), port) for port in range(20000, 20000 + num_nodes)]))' >nodes.txt

View File

@ -1,10 +1,10 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
set -o errexit set -o errexit
cat << EOF > aws_e2e_setup.json cat <<EOF >aws_e2e_setup.json
{ {
"iam_auth_ecs_account" : "${iam_auth_ecs_account}", "iam_auth_ecs_account" : "${iam_auth_ecs_account}",
"iam_auth_ecs_secret_access_key" : "${iam_auth_ecs_secret_access_key}", "iam_auth_ecs_secret_access_key" : "${iam_auth_ecs_secret_access_key}",

View File

@ -1,10 +1,10 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
set -o errexit set -o errexit
cat << EOF > $HOME/azure_e2e_config.json cat <<EOF >$HOME/azure_e2e_config.json
{ {
"tD548GwE1@outlook.com" : "${oidc_azure_test_user_account_one_secret}", "tD548GwE1@outlook.com" : "${oidc_azure_test_user_account_one_secret}",
"tD548GwE2@outlook.com" : "${oidc_azure_test_user_account_two_secret}", "tD548GwE2@outlook.com" : "${oidc_azure_test_user_account_two_secret}",
@ -24,12 +24,12 @@ cat << EOF > $HOME/azure_e2e_config.json
"oidc_azure_managed_identity_api_version": "${oidc_azure_managed_identity_api_version}" "oidc_azure_managed_identity_api_version": "${oidc_azure_managed_identity_api_version}"
} }
EOF EOF
cat << EOF > $HOME/oidc_azure_container_key cat <<EOF >$HOME/oidc_azure_container_key
${oidc_azure_container_key} ${oidc_azure_container_key}
EOF EOF
# EVG project variables do not preserve line breaks so we store them as base64 and decode here # EVG project variables do not preserve line breaks so we store them as base64 and decode here
sed s/[[:space:]]//g $HOME/oidc_azure_container_key | base64 --decode > $HOME/azure_remote_key sed s/[[:space:]]//g $HOME/oidc_azure_container_key | base64 --decode >$HOME/azure_remote_key
# Clean up temp file # Clean up temp file
rm -f $HOME/oidc_azure_container_key rm -f $HOME/oidc_azure_container_key

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
set -o errexit set -o errexit

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -7,7 +7,7 @@ set -o errexit
# Create the config file, which will contain the GCE project/zone information along with # Create the config file, which will contain the GCE project/zone information along with
# the expected audience that will appear on the VM's ID token. # the expected audience that will appear on the VM's ID token.
cat << EOF > $HOME/gce_vm_config.json cat <<EOF >$HOME/gce_vm_config.json
{ {
"audience" : "${oidc_gcp_vm_id_token_audience}", "audience" : "${oidc_gcp_vm_id_token_audience}",
"projectID" : "${oidc_gcp_project_id}", "projectID" : "${oidc_gcp_project_id}",
@ -19,7 +19,7 @@ EOF
# Create the SSH key file. Note that the SSH key has been base64 encoded and stored into an EVG # Create the SSH key file. Note that the SSH key has been base64 encoded and stored into an EVG
# environment variable, so it is first trimmed of any whitespace via sed and base64 decoded before # environment variable, so it is first trimmed of any whitespace via sed and base64 decoded before
# being output to the file. # being output to the file.
echo ${oidc_gcp_ssh_key} | sed "s/[[:space:]]//g" | base64 --decode > $HOME/gcp_ssh_key echo ${oidc_gcp_ssh_key} | sed "s/[[:space:]]//g" | base64 --decode >$HOME/gcp_ssh_key
# Reduce SSH keyfile privileges so that it is secure enough for OpenSSH. # Reduce SSH keyfile privileges so that it is secure enough for OpenSSH.
chmod 600 $HOME/gcp_ssh_key chmod 600 $HOME/gcp_ssh_key
@ -34,7 +34,7 @@ ls -al $HOME/gcp_ssh_key
# The contents of this file are expected to exist in base64 encoded format in # The contents of this file are expected to exist in base64 encoded format in
# $oidc_gcp_service_account_key, so the same steps are taken as above before dumping it into a # $oidc_gcp_service_account_key, so the same steps are taken as above before dumping it into a
# newly-created JSON file. # newly-created JSON file.
echo ${oidc_gcp_service_account_key} | sed "s/[[:space:]]//g" | base64 --decode > ${GOOGLE_APPLICATION_CREDENTIALS} echo ${oidc_gcp_service_account_key} | sed "s/[[:space:]]//g" | base64 --decode >${GOOGLE_APPLICATION_CREDENTIALS}
chmod 600 ${GOOGLE_APPLICATION_CREDENTIALS} chmod 600 ${GOOGLE_APPLICATION_CREDENTIALS}
ls -al ${GOOGLE_APPLICATION_CREDENTIALS} ls -al ${GOOGLE_APPLICATION_CREDENTIALS}

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -14,10 +14,10 @@ echo "Cleaning up OIDC GCP test artifacts"
# Delete the GCP VM specified in gce_vm_info.json if GOOGLE_APPLICATION_CREDENTIALS is set, points # Delete the GCP VM specified in gce_vm_info.json if GOOGLE_APPLICATION_CREDENTIALS is set, points
# to a file, and the GCE config and VM info files exist. # to a file, and the GCE config and VM info files exist.
if [ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" ] \ if [ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" ] &&
&& [ -f "${GOOGLE_APPLICATION_CREDENTIALS}" ] \ [ -f "${GOOGLE_APPLICATION_CREDENTIALS}" ] &&
&& [ -f "${HOME}/gce_vm_config.json" ] \ [ -f "${HOME}/gce_vm_config.json" ] &&
&& [ -f "${HOME}/gce_vm_info.json" ]; then [ -f "${HOME}/gce_vm_info.json" ]; then
# Install google-cloud-compute so that the script can run. # Install google-cloud-compute so that the script can run.
$python -m pip install google-cloud-compute $python -m pip install google-cloud-compute
$python src/mongo/db/modules/enterprise/jstests/external_auth_oidc_gcp/lib/gce_vm_manager.py delete --config_file $HOME/gce_vm_config.json --service_account_key_file ${GOOGLE_APPLICATION_CREDENTIALS} --output_file $HOME/gce_vm_info.json $python src/mongo/db/modules/enterprise/jstests/external_auth_oidc_gcp/lib/gce_vm_manager.py delete --config_file $HOME/gce_vm_config.json --service_account_key_file ${GOOGLE_APPLICATION_CREDENTIALS} --output_file $HOME/gce_vm_info.json

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -6,7 +6,7 @@ cd src
set -o errexit set -o errexit
# Should output contents to new file in home directory. # Should output contents to new file in home directory.
cat << EOF > $HOME/oidc_e2e_setup.json cat <<EOF >$HOME/oidc_e2e_setup.json
{ {
"testserversecurityone@ping-test.com" : "${oidc_ping_test_user_account_one_secret}", "testserversecurityone@ping-test.com" : "${oidc_ping_test_user_account_one_secret}",
"testserversecuritytwo@ping-test.com" : "${oidc_ping_test_user_account_two_secret}", "testserversecuritytwo@ping-test.com" : "${oidc_ping_test_user_account_two_secret}",

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
# Only run this script for the external_auth_oidc_azure task. # Only run this script for the external_auth_oidc_azure task.

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -6,9 +6,9 @@ cd src
set -eou pipefail set -eou pipefail
# Only run on unit test tasks so we don't target mongod binaries from cores. # Only run on unit test tasks so we don't target mongod binaries from cores.
if [ "${task_name}" != "run_dbtest" ] \ if [ "${task_name}" != "run_dbtest" ] &&
&& [[ ${task_name} != integration_tests* ]] \ [[ ${task_name} != integration_tests* ]] &&
&& [[ "${task_name}" != unit_test_group*_no_sandbox ]]; then [[ "${task_name}" != unit_test_group*_no_sandbox ]]; then
echo "Not gathering failed unittests binaries as this is not a unittest task: ${task_name}" echo "Not gathering failed unittests binaries as this is not a unittest task: ${task_name}"
exit 0 exit 0
fi fi
@ -17,14 +17,14 @@ unittest_bin_dir=dist-unittests/bin
mkdir -p $unittest_bin_dir || true mkdir -p $unittest_bin_dir || true
# Find all core files # Find all core files
core_files=$(/usr/bin/find -H . \( -name "dump_*.core" -o -name "*.mdmp" \) 2> /dev/null) core_files=$(/usr/bin/find -H . \( -name "dump_*.core" -o -name "*.mdmp" \) 2>/dev/null)
while read -r core_file; do while read -r core_file; do
# A core file name does not always have the executable name that generated it. # A core file name does not always have the executable name that generated it.
# See http://stackoverflow.com/questions/34801353/core-dump-filename-gets-thread-name-instead-of-executable-name-with-core-pattern # See http://stackoverflow.com/questions/34801353/core-dump-filename-gets-thread-name-instead-of-executable-name-with-core-pattern
# On platforms with GDB, we get the binary name from core file # On platforms with GDB, we get the binary name from core file
gdb=/opt/mongodbtoolchain/v5/bin/gdb gdb=/opt/mongodbtoolchain/v5/bin/gdb
if [ -f $gdb ]; then if [ -f $gdb ]; then
binary_file=$($gdb -batch --quiet -ex "core $core_file" 2> /dev/null | grep "Core was generated" | cut -f2 -d "\`" | cut -f1 -d "'" | cut -f1 -d " ") binary_file=$($gdb -batch --quiet -ex "core $core_file" 2>/dev/null | grep "Core was generated" | cut -f2 -d "\`" | cut -f1 -d "'" | cut -f1 -d " ")
binary_file_locations=$binary_file binary_file_locations=$binary_file
else else
echo "Checking core file '$core_file'" echo "Checking core file '$core_file'"
@ -34,7 +34,7 @@ while read -r core_file; do
# Locate the binary file. Since the base file name might be truncated, the find # Locate the binary file. Since the base file name might be truncated, the find
# may return more than 1 file. # may return more than 1 file.
if [ "$binary_file" != "" ]; then if [ "$binary_file" != "" ]; then
binary_file_locations=$(/usr/bin/find -H . -executable -name "$binary_file*${exe}" 2> /dev/null) binary_file_locations=$(/usr/bin/find -H . -executable -name "$binary_file*${exe}" 2>/dev/null)
fi fi
fi fi
@ -84,7 +84,7 @@ while read -r core_file; do
fi fi
done done
done <<< "${core_files}" done <<<"${core_files}"
# Copy debug symbols for dynamic builds # Copy debug symbols for dynamic builds
lib_dir=bazel-bin/install/lib lib_dir=bazel-bin/install/lib

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -14,11 +14,11 @@ while read -r output; do
echo "Running out of space \"$partition ($usep%)\" on $(hostname) as on $(date)" echo "Running out of space \"$partition ($usep%)\" on $(hostname) as on $(date)"
HAS_FULL_DISK=true HAS_FULL_DISK=true
fi fi
done <<< "$FILESYSTEMS" done <<<"$FILESYSTEMS"
if $HAS_FULL_DISK; then if $HAS_FULL_DISK; then
# print all files that are above one megabyte sorted # print all files that are above one megabyte sorted
du -cha / 2> /dev/null | grep -E "^[0-9]+(\.[0-9]+)?[G|M|T]" | sort -h du -cha / 2>/dev/null | grep -E "^[0-9]+(\.[0-9]+)?[G|M|T]" | sort -h
else else
echo "No full partitions found, skipping" echo "No full partitions found, skipping"
fi fi

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
set +o errexit set +o errexit

View File

@ -1,9 +1,9 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
cd src cd src
cat > mci.buildlogger << END_OF_CREDS cat >mci.buildlogger <<END_OF_CREDS
slavename='${slave}' slavename='${slave}'
passwd='${passwd}' passwd='${passwd}'
builder='${build_variant}_${project}' builder='${build_variant}_${project}'

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
set +o errexit set +o errexit

View File

@ -1,10 +1,10 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
cd src cd src
# Create the Evergreen API credentials # Create the Evergreen API credentials
cat > .evergreen.yml << END_OF_CREDS cat >.evergreen.yml <<END_OF_CREDS
api_server_host: https://evergreen.mongodb.com/api api_server_host: https://evergreen.mongodb.com/api
api_key: "${evergreen_api_key}" api_key: "${evergreen_api_key}"
user: "${evergreen_api_user}" user: "${evergreen_api_user}"

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
if [ -z "${files}" ]; then if [ -z "${files}" ]; then

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
cd src cd src

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
cd src cd src
@ -9,11 +9,11 @@ set -o errexit
# For patch builds gather the modified patch files. # For patch builds gather the modified patch files.
if [ "${is_patch}" = "true" ]; then if [ "${is_patch}" = "true" ]; then
# Get list of patched files # Get list of patched files
git diff HEAD --name-only >> patch_files.txt git diff HEAD --name-only >>patch_files.txt
if [ -d src/mongo/db/modules/enterprise ]; then if [ -d src/mongo/db/modules/enterprise ]; then
pushd src/mongo/db/modules/enterprise pushd src/mongo/db/modules/enterprise
# Update the patch_files.txt in the mongo repo. # Update the patch_files.txt in the mongo repo.
git diff HEAD --name-only >> ~1/patch_files.txt git diff HEAD --name-only >>~1/patch_files.txt
popd popd
fi fi
fi fi

View File

@ -1,14 +1,14 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
cd src cd src
set -o errexit set -o errexit
cat << EOF > notary_env.sh cat <<EOF >notary_env.sh
export NOTARY_TOKEN=${signing_auth_token_70} export NOTARY_TOKEN=${signing_auth_token_70}
export BARQUE_USERNAME=${barque_user} export BARQUE_USERNAME=${barque_user}
export BARQUE_API_KEY=${barque_api_key} export BARQUE_API_KEY=${barque_api_key}
EOF EOF
echo "${signing_auth_token_70}" > signing_auth_token echo "${signing_auth_token_70}" >signing_auth_token

View File

@ -1,17 +1,17 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
proc_list="(java|lein|mongo|python|_test$|_test\.exe$)" proc_list="(java|lein|mongo|python|_test$|_test\.exe$)"
if [ "Windows_NT" = "$OS" ]; then if [ "Windows_NT" = "$OS" ]; then
get_pids() { get_pids() {
proc_pids=$(tasklist /fo:csv \ proc_pids=$(tasklist /fo:csv |
| awk -F'","' '{x=$1; gsub("\"","",x); print $2, x}' \ awk -F'","' '{x=$1; gsub("\"","",x); print $2, x}' |
| grep -iE $1 \ grep -iE $1 |
| cut -f1 -d ' ') cut -f1 -d ' ')
} }
get_process_info() { get_process_info() {
proc_name="" proc_name=""
proc_info=$(wmic process where "ProcessId=\"$1\"" get "Name,ProcessId,ThreadCount" /format:csv 2> /dev/null | grep $1) proc_info=$(wmic process where "ProcessId=\"$1\"" get "Name,ProcessId,ThreadCount" /format:csv 2>/dev/null | grep $1)
if [ ! -z $proc_info ]; then if [ ! -z $proc_info ]; then
proc_name=$(echo $proc_info | cut -f2 -d ',') proc_name=$(echo $proc_info | cut -f2 -d ',')
proc_threads=$(echo $proc_info | cut -f4 -d ',') proc_threads=$(echo $proc_info | cut -f4 -d ',')

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
# Since the macros 'private_key_remote' and 'private_key_file' are not always defined # Since the macros 'private_key_remote' and 'private_key_file' are not always defined
@ -9,7 +9,7 @@ fi
if [ ! -z "${private_key_remote}" ] && [ ! -z "${private_key_file}" ]; then if [ ! -z "${private_key_remote}" ] && [ ! -z "${private_key_file}" ]; then
mkdir -p ~/.ssh mkdir -p ~/.ssh
private_key_file=$(eval echo "$private_key_file") private_key_file=$(eval echo "$private_key_file")
echo -n "${private_key_remote}" > ${private_key_file} echo -n "${private_key_remote}" >${private_key_file}
chmod 0600 ${private_key_file} chmod 0600 ${private_key_file}
fi fi
@ -28,19 +28,19 @@ if [ $aws_profile = "default" ]; then
else else
aws_profile_config="[profile $aws_profile]" aws_profile_config="[profile $aws_profile]"
fi fi
cat << EOF >> ~/.aws/config cat <<EOF >>~/.aws/config
$aws_profile_config $aws_profile_config
region = us-east-1 region = us-east-1
EOF EOF
# The profile in the credentials file is specified as [<profile>]. # The profile in the credentials file is specified as [<profile>].
cat << EOF >> ~/.aws/credentials cat <<EOF >>~/.aws/credentials
[$aws_profile] [$aws_profile]
aws_access_key_id = ${aws_key_remote} aws_access_key_id = ${aws_key_remote}
aws_secret_access_key = ${aws_secret_remote} aws_secret_access_key = ${aws_secret_remote}
EOF EOF
cat << EOF > ~/.boto cat <<EOF >~/.boto
[Boto] [Boto]
https_validate_certificates = False https_validate_certificates = False
EOF EOF

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
cd src cd src

View File

@ -3,20 +3,20 @@ set -o verbose
# On Windows we can use typeperf.exe to dump performance counters. # On Windows we can use typeperf.exe to dump performance counters.
if [ "Windows_NT" = "$OS" ]; then if [ "Windows_NT" = "$OS" ]; then
typeperf -qx PhysicalDisk | grep Disk | grep -v _Total > disk_counters.txt typeperf -qx PhysicalDisk | grep Disk | grep -v _Total >disk_counters.txt
typeperf -cf disk_counters.txt -si 5 -o mongo-diskstats typeperf -cf disk_counters.txt -si 5 -o mongo-diskstats
# Linux: iostat -t option for timestamp. # Linux: iostat -t option for timestamp.
elif iostat -tdmx > /dev/null 2>&1; then elif iostat -tdmx >/dev/null 2>&1; then
iostat -tdmx 5 > mongo-diskstats iostat -tdmx 5 >mongo-diskstats
# OSX: Simulate the iostat timestamp. # OSX: Simulate the iostat timestamp.
elif iostat -d > /dev/null 2>&1; then elif iostat -d >/dev/null 2>&1; then
iostat -d -w 5 | while IFS= read -r line; do printf '%s %s\n' "$(date +'%m/%d/%Y %H:%M:%S')" "$line" >> mongo-diskstats; done iostat -d -w 5 | while IFS= read -r line; do printf '%s %s\n' "$(date +'%m/%d/%Y %H:%M:%S')" "$line" >>mongo-diskstats; done
# Check if vmstat -t is available. # Check if vmstat -t is available.
elif vmstat -td > /dev/null 2>&1; then elif vmstat -td >/dev/null 2>&1; then
vmstat -td 5 > mongo-diskstats vmstat -td 5 >mongo-diskstats
# Check if vmstat -T d is available. # Check if vmstat -T d is available.
elif vmstat -T d > /dev/null 2>&1; then elif vmstat -T d >/dev/null 2>&1; then
vmstat -T d 5 > mongo-diskstats vmstat -T d 5 >mongo-diskstats
else else
printf "Cannot collect mongo-diskstats on this platform\n" printf "Cannot collect mongo-diskstats on this platform\n"
fi fi

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
cd src cd src

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
cd src cd src

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
set -o errexit set -o errexit

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
if [ "$(uname)" != "Linux" ] && [ "$(uname)" != "Darwin" ]; then if [ "$(uname)" != "Linux" ] && [ "$(uname)" != "Darwin" ]; then

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude_python.sh" . "$DIR/../prelude_python.sh"
set -o errexit set -o errexit

View File

@ -1,7 +1,7 @@
# exit immediately if virtualenv is not found # exit immediately if virtualenv is not found
set -o errexit set -o errexit
evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)/.." evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.."
. "$evergreen_dir/prelude_workdir.sh" . "$evergreen_dir/prelude_workdir.sh"
. "$evergreen_dir/prelude_python.sh" . "$evergreen_dir/prelude_python.sh"
@ -49,7 +49,7 @@ fi
"$python_loc" -m venv "$venv_dir" "$python_loc" -m venv "$venv_dir"
# Adding README file for using this venv locally # Adding README file for using this venv locally
cat << EOF >> venv_readme.txt cat <<EOF >>venv_readme.txt
This is an archive of the Python venv generated by this Evergreen build. This is an archive of the Python venv generated by this Evergreen build.
You can use it locally to avoid needing to manually set up the Python environment. You can use it locally to avoid needing to manually set up the Python environment.
@ -60,11 +60,11 @@ echo "Updating virtual env directory in activate script"
pushd venv; venv_dir=\$(pwd); popd pushd venv; venv_dir=\$(pwd); popd
EOF EOF
if [ "Windows_NT" = "$OS" ]; then if [ "Windows_NT" = "$OS" ]; then
cat << EOF >> venv_readme.txt cat <<EOF >>venv_readme.txt
sed -i -e "s:VIRTUAL_ENV=\".*\":VIRTUAL_ENV=\"\$venv_dir\":" "\$venv_dir/Scripts/activate" sed -i -e "s:VIRTUAL_ENV=\".*\":VIRTUAL_ENV=\"\$venv_dir\":" "\$venv_dir/Scripts/activate"
EOF EOF
else else
cat << EOF >> venv_readme.txt cat <<EOF >>venv_readme.txt
sed -i -e "s:VIRTUAL_ENV=\".*\":VIRTUAL_ENV=\"\$venv_dir\":" "\$venv_dir/bin/activate" sed -i -e "s:VIRTUAL_ENV=\".*\":VIRTUAL_ENV=\"\$venv_dir\":" "\$venv_dir/bin/activate"
echo "Adding back python symlinks" echo "Adding back python symlinks"
@ -133,4 +133,4 @@ fi
cd .. cd ..
python -m pip freeze > pip-requirements.txt python -m pip freeze >pip-requirements.txt

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
cd src cd src

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
cd src cd src

View File

@ -1,10 +1,10 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_80}" >> "signing-envfile" echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_80}" >>"signing-envfile"
echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_80}" >> "signing-envfile" echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_80}" >>"signing-envfile"
set -o errexit set -o errexit
set -o verbose set -o verbose
@ -20,7 +20,7 @@ shasum -a 256 $crypt_file_name | tee $crypt_file_name.sha256
md5sum $crypt_file_name | tee $crypt_file_name.md5 md5sum $crypt_file_name | tee $crypt_file_name.md5
# signing crypt linux artifact with gpg # signing crypt linux artifact with gpg
cat << EOF >> gpg_signing_commands.sh cat <<EOF >>gpg_signing_commands.sh
gpgloader # loading gpg keys. gpgloader # loading gpg keys.
gpg --yes -v --armor -o $crypt_file_name.sig --detach-sign $crypt_file_name gpg --yes -v --armor -o $crypt_file_name.sig --detach-sign $crypt_file_name
EOF EOF

View File

@ -1,10 +1,10 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_80}" >> "signing-envfile" echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_80}" >>"signing-envfile"
echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_80}" >> "signing-envfile" echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_80}" >>"signing-envfile"
set -o errexit set -o errexit
set -o verbose set -o verbose
@ -37,7 +37,7 @@ gen_checksums mongodb-src-$src_suffix.$long_ext
gen_checksums mongodb-cryptd-$push_name-$push_arch-$suffix.$ext gen_checksums mongodb-cryptd-$push_name-$push_arch-$suffix.$ext
# signing linux artifacts with gpg # signing linux artifacts with gpg
cat << 'EOF' > gpg_signing_commands.sh cat <<'EOF' >gpg_signing_commands.sh
gpgloader # loading gpg keys. gpgloader # loading gpg keys.
function sign(){ function sign(){
if [ -e $1 ] if [ -e $1 ]
@ -50,7 +50,7 @@ function sign(){
EOF EOF
cat << EOF >> gpg_signing_commands.sh cat <<EOF >>gpg_signing_commands.sh
sign mongodb-$push_name-$push_arch-$suffix.$ext sign mongodb-$push_name-$push_arch-$suffix.$ext
sign mongodb-$push_name-$push_arch-debugsymbols-$suffix.$ext sign mongodb-$push_name-$push_arch-debugsymbols-$suffix.$ext
sign mongodb-src-$src_suffix.$long_ext sign mongodb-src-$src_suffix.$long_ext

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
if [ "${push_name}" != "windows" ]; then if [ "${push_name}" != "windows" ]; then
@ -7,8 +7,8 @@ fi
cd src cd src
echo "GRS_CONFIG_USER1_USERNAME=${garasign_jsign_username}" >> "signing-envfile" echo "GRS_CONFIG_USER1_USERNAME=${garasign_jsign_username}" >>"signing-envfile"
echo "GRS_CONFIG_USER1_PASSWORD=${garasign_jsign_password}" >> "signing-envfile" echo "GRS_CONFIG_USER1_PASSWORD=${garasign_jsign_password}" >>"signing-envfile"
set -o errexit set -o errexit
set -o verbose set -o verbose
@ -18,7 +18,7 @@ cp bazel-bin/src/mongo/installer/msi/mongodb-win32-x86_64-windows-${version}.msi
if [ "${is_patch}" != "true" ]; then if [ "${is_patch}" != "true" ]; then
# signing windows artifacts with jsign # signing windows artifacts with jsign
cat << 'EOF' > jsign_signing_commands.sh cat <<'EOF' >jsign_signing_commands.sh
function sign(){ function sign(){
if [ -e $1 ] if [ -e $1 ]
then then
@ -28,7 +28,7 @@ function sign(){
fi fi
} }
EOF EOF
cat << EOF >> jsign_signing_commands.sh cat <<EOF >>jsign_signing_commands.sh
sign $msi_filename sign $msi_filename
EOF EOF

View File

@ -1,6 +1,6 @@
cd src cd src
# Find all core files and move to src # Find all core files and move to src
core_files=$(/usr/bin/find -H .. \( -name "*.core" -o -name "*.mdmp" \) 2> /dev/null) core_files=$(/usr/bin/find -H .. \( -name "*.core" -o -name "*.mdmp" \) 2>/dev/null)
for core_file in $core_files; do for core_file in $core_files; do
base_name=$(echo $core_file | sed "s/.*\///") base_name=$(echo $core_file | sed "s/.*\///")
# Move file if it does not already exist # Move file if it does not already exist

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -59,11 +59,11 @@ if [ -z "${build_patch_id}" ] || [ -z "${reuse_compile_from}" ] || [ "${is_patch
uarch=$(uname -p) uarch=$(uname -p)
os=$(uname -r) os=$(uname -r)
json="{ \"version\": \"${version}\", \"gitVersion\": \"${revision}\", \"uarch\": \"$uarch\", \"os\": \"$os\" }" json="{ \"version\": \"${version}\", \"gitVersion\": \"${revision}\", \"uarch\": \"$uarch\", \"os\": \"$os\" }"
echo $json | jq '.' > mongodb/atlas_info.json echo $json | jq '.' >mongodb/atlas_info.json
# Add custom run_validate_collections.js wrapper # Add custom run_validate_collections.js wrapper
mv jstests/hooks/run_validate_collections.js jstests/hooks/run_validate_collections.actual.js mv jstests/hooks/run_validate_collections.js jstests/hooks/run_validate_collections.actual.js
cat << EOF > jstests/hooks/run_validate_collections.js cat <<EOF >jstests/hooks/run_validate_collections.js
print("NOTE: run_validate_collections.js will skip the oplog!"); print("NOTE: run_validate_collections.js will skip the oplog!");
TestData = { skipValidationNamespaces: ['local.oplog.rs'] }; TestData = { skipValidationNamespaces: ['local.oplog.rs'] };
await import("jstests/hooks/run_validate_collections.actual.js"); await import("jstests/hooks/run_validate_collections.actual.js");

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -20,7 +20,7 @@ promote_revision=$(cat task_data.json | jq -r ".revision")
artifact_address="https://internal-downloads.mongodb.com/server-custom-builds/${promote_project_id}/${promote_version_id}/${promote_build_variant}/mongo-${promote_build_id}.tgz" artifact_address="https://internal-downloads.mongodb.com/server-custom-builds/${promote_project_id}/${promote_version_id}/${promote_build_variant}/mongo-${promote_build_id}.tgz"
cat << EOT > ./promote-expansions.yml cat <<EOT >./promote-expansions.yml
promote_project_id: "$promote_project_id" promote_project_id: "$promote_project_id"
promote_version_id: "$promote_version_id" promote_version_id: "$promote_version_id"
promote_build_id: "$promote_build_id" promote_build_id: "$promote_build_id"

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -15,42 +15,42 @@ if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
if [[ "$PWD" != "$(cat Z:/bazel_tmp/mci_path)" ]]; then if [[ "$PWD" != "$(cat Z:/bazel_tmp/mci_path)" ]]; then
echo "Clearing bazel output root from previous task mci '$(cat Z:/bazel_tmp/mci_path)'" echo "Clearing bazel output root from previous task mci '$(cat Z:/bazel_tmp/mci_path)'"
rm -rf Z:/bazel_tmp/* || true rm -rf Z:/bazel_tmp/* || true
echo $PWD > Z:/bazel_tmp/mci_path echo $PWD >Z:/bazel_tmp/mci_path
fi fi
# Z:/ path is necessary to avoid running into MSVC's file length limit, # Z:/ path is necessary to avoid running into MSVC's file length limit,
# see https://jira.mongodb.org/browse/DEVPROD-11126 # see https://jira.mongodb.org/browse/DEVPROD-11126
abs_path=$(cygpath -w "$TMPDIR" | tr '\\' '/') abs_path=$(cygpath -w "$TMPDIR" | tr '\\' '/')
echo "startup --output_user_root=Z:/bazel_tmp" > .bazelrc.evergreen echo "startup --output_user_root=Z:/bazel_tmp" >.bazelrc.evergreen
echo "common --action_env=TMP=Z:/bazel_tmp" >> .bazelrc.evergreen echo "common --action_env=TMP=Z:/bazel_tmp" >>.bazelrc.evergreen
echo "common --action_env=TEMP=Z:/bazel_tmp" >> .bazelrc.evergreen echo "common --action_env=TEMP=Z:/bazel_tmp" >>.bazelrc.evergreen
echo "BAZELISK_HOME=${abs_path}/bazelisk_home" >> .bazeliskrc echo "BAZELISK_HOME=${abs_path}/bazelisk_home" >>.bazeliskrc
# echo "common --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" >> .bazelrc.git # echo "common --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" >> .bazelrc.git
echo "common --define GIT_COMMIT_HASH=nogitversion" >> .bazelrc.git echo "common --define GIT_COMMIT_HASH=nogitversion" >>.bazelrc.git
else else
echo "startup --output_user_root=${TMPDIR}/bazel-output-root" > .bazelrc.evergreen echo "startup --output_user_root=${TMPDIR}/bazel-output-root" >.bazelrc.evergreen
echo "BAZELISK_HOME=${TMPDIR}/bazelisk_home" >> .bazeliskrc echo "BAZELISK_HOME=${TMPDIR}/bazelisk_home" >>.bazeliskrc
echo "common --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" >> .bazelrc.git echo "common --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" >>.bazelrc.git
fi fi
if [[ "${evergreen_remote_exec}" != "on" ]]; then if [[ "${evergreen_remote_exec}" != "on" ]]; then
# Temporarily disable remote exec and only use remote cache # Temporarily disable remote exec and only use remote cache
echo "common --remote_executor=" >> .bazelrc.evergreen echo "common --remote_executor=" >>.bazelrc.evergreen
echo "common --modify_execution_info=.*=+no-remote-exec" >> .bazelrc.evergreen echo "common --modify_execution_info=.*=+no-remote-exec" >>.bazelrc.evergreen
echo "common --jobs=auto" >> .bazelrc.evergreen echo "common --jobs=auto" >>.bazelrc.evergreen
fi fi
uri="https://spruce.mongodb.com/task/${task_id:?}?execution=${execution:?}" uri="https://spruce.mongodb.com/task/${task_id:?}?execution=${execution:?}"
echo "common --tls_client_certificate=./engflow.cert" >> .bazelrc.evergreen echo "common --tls_client_certificate=./engflow.cert" >>.bazelrc.evergreen
echo "common --tls_client_key=./engflow.key" >> .bazelrc.evergreen echo "common --tls_client_key=./engflow.key" >>.bazelrc.evergreen
echo "common --bes_keywords=engflow:CiCdPipelineName=${build_variant:?}" >> .bazelrc.evergreen echo "common --bes_keywords=engflow:CiCdPipelineName=${build_variant:?}" >>.bazelrc.evergreen
echo "common --bes_keywords=engflow:CiCdJobName=${task_name:?}" >> .bazelrc.evergreen echo "common --bes_keywords=engflow:CiCdJobName=${task_name:?}" >>.bazelrc.evergreen
echo "common --bes_keywords=engflow:CiCdUri=${uri:?}" >> .bazelrc.evergreen echo "common --bes_keywords=engflow:CiCdUri=${uri:?}" >>.bazelrc.evergreen
echo "common --bes_keywords=evg:project=${project:?}" >> .bazelrc.evergreen echo "common --bes_keywords=evg:project=${project:?}" >>.bazelrc.evergreen
echo "common --remote_upload_local_results=True" >> .bazelrc.evergreen echo "common --remote_upload_local_results=True" >>.bazelrc.evergreen
echo "common --test_output=summary" >> .bazelrc.evergreen echo "common --test_output=summary" >>.bazelrc.evergreen
# Disable remote execution in evergreen only since it runs on every PR, but we still # Disable remote execution in evergreen only since it runs on every PR, but we still
# want it to be fast on workstations # want it to be fast on workstations
echo "coverage --config=no-remote-exec" >> .bazelrc.evergreen echo "coverage --config=no-remote-exec" >>.bazelrc.evergreen

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -4,7 +4,7 @@
# Usage: # Usage:
# bash get_all_resmoke_suite_configs.sh # bash get_all_resmoke_suite_configs.sh
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -21,4 +21,4 @@ BAZEL_BINARY=$(bazel_get_binary_path)
# f.path for f in target.files.to_list() -> the path to the config file, like bazel-out/k8-fastbuild/bin/buildscripts/resmokeconfig/core.yml # f.path for f in target.files.to_list() -> the path to the config file, like bazel-out/k8-fastbuild/bin/buildscripts/resmokeconfig/core.yml
${BAZEL_BINARY} cquery ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \ ${BAZEL_BINARY} cquery ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \
--define=MONGO_VERSION=${version} ${patch_compile_flags} "kind(resmoke_config, //...)" \ --define=MONGO_VERSION=${version} ${patch_compile_flags} "kind(resmoke_config, //...)" \
--output=starlark --starlark:expr "': '.join([str(target.label).replace('@@','')] + [f.path for f in target.files.to_list()])" > resmoke_suite_configs.yml --output=starlark --starlark:expr "': '.join([str(target.label).replace('@@','')] + [f.path for f in target.files.to_list()])" >resmoke_suite_configs.yml

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -6,7 +6,7 @@
# Required environment variables: # Required environment variables:
# * ${suite} - Resmoke bazel target, like //buildscripts/resmokeconfig:core # * ${suite} - Resmoke bazel target, like //buildscripts/resmokeconfig:core
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -18,4 +18,4 @@ source ./evergreen/bazel_utility_functions.sh
BAZEL_BINARY=$(bazel_get_binary_path) BAZEL_BINARY=$(bazel_get_binary_path)
echo "suite_config: $(${BAZEL_BINARY} cquery ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \ echo "suite_config: $(${BAZEL_BINARY} cquery ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \
--define=MONGO_VERSION=${version} ${patch_compile_flags} ${suite}_config --output files)" > suite_config_expansion.yml --define=MONGO_VERSION=${version} ${patch_compile_flags} ${suite}_config --output files)" >suite_config_expansion.yml

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

View File

@ -1,4 +1,4 @@
set -euo pipefail set -euo pipefail
cd jepsen/docker cd jepsen/docker
./bin/up -n 9 -d 2>&1 > docker.log ./bin/up -n 9 -d 2>&1 >docker.log

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
# this file does not use set -euo pipefail because we determine test success or # this file does not use set -euo pipefail because we determine test success or
@ -71,7 +71,7 @@ for i in {1..30}; do
done done
# Merge all jepsen_test_${i}.log into a single file # Merge all jepsen_test_${i}.log into a single file
cat src/jepsen-mongodb/jepsen_test_*.log > src/jepsen-mongodb/jepsen_${task_name}_${execution}.log cat src/jepsen-mongodb/jepsen_test_*.log >src/jepsen-mongodb/jepsen_${task_name}_${execution}.log
end_time=$(date +%s) end_time=$(date +%s)
elapsed_secs=$((end_time - start_time)) elapsed_secs=$((end_time - start_time))

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh" . "$DIR/../prelude.sh"
cd src cd src

View File

@ -16,10 +16,10 @@ else
final_exit_code=1 final_exit_code=1
else else
# If the failure is due to setup, then this is considered a system failure. # If the failure is due to setup, then this is considered a system failure.
echo $grep_exit_code > jepsen_system_failure_${task_name}_${execution} echo $grep_exit_code >jepsen_system_failure_${task_name}_${execution}
exit 0 exit 0
fi fi
fi fi
# Create report.json # Create report.json
echo "{\"failures\": $failures, \"results\": [{\"status\": $status, \"exit_code\": $final_exit_code, \"test_file\": \"${task_name}\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > ../report.json echo "{\"failures\": $failures, \"results\": [{\"status\": $status, \"exit_code\": $final_exit_code, \"test_file\": \"${task_name}\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" >../report.json
exit $final_exit_code exit $final_exit_code

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src/jepsen-mongodb cd src/jepsen-mongodb

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src/jepsen-mongodb cd src/jepsen-mongodb
@ -36,8 +36,8 @@ lein run test --test ${jepsen_test_name} \
${jepsen_storage_engine} \ ${jepsen_storage_engine} \
${jepsen_time_limit} \ ${jepsen_time_limit} \
${jepsen_write_concern} \ ${jepsen_write_concern} \
2>&1 \ 2>&1 |
| tee jepsen_${task_name}_${execution}.log tee jepsen_${task_name}_${execution}.log
end_time=$(date +%s) end_time=$(date +%s)
elapsed_secs=$((end_time - start_time)) elapsed_secs=$((end_time - start_time))
. ../evergreen/jepsen_report.sh . ../evergreen/jepsen_report.sh

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src/jstestfuzz cd src/jstestfuzz

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
set -evo pipefail set -evo pipefail

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -6,11 +6,11 @@ cd src
set -o errexit set -o errexit
set -o verbose set -o verbose
if command -v docker > /dev/null 2>&1; then if command -v docker >/dev/null 2>&1; then
echo "Docker is installed, using docker" echo "Docker is installed, using docker"
CONTAINER_RUNTIME=docker CONTAINER_RUNTIME=docker
elif command -v podman > /dev/null 2>&1; then elif command -v podman >/dev/null 2>&1; then
echo "Podman is installed, using podman" echo "Podman is installed, using podman"
CONTAINER_RUNTIME=podman CONTAINER_RUNTIME=podman
else else
@ -18,6 +18,6 @@ else
exit 1 exit 1
fi fi
echo "CONTAINER_RUNTIME: ${CONTAINER_RUNTIME}" >> expansions.yml echo "CONTAINER_RUNTIME: ${CONTAINER_RUNTIME}" >>expansions.yml
aws ecr get-login-password --region us-east-1 | $CONTAINER_RUNTIME login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com aws ecr get-login-password --region us-east-1 | $CONTAINER_RUNTIME login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
@ -13,9 +13,9 @@ CONTAINER_INPUT_DIR="/app"
podman pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest podman pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest
# Run parse-jsfiles on 50 files at a time with 32 processes in parallel. # Run parse-jsfiles on 50 files at a time with 32 processes in parallel.
# Skip javascript files in third_party directory # Skip javascript files in third_party directory
find "jstests" "src/mongo/db/modules/enterprise" -path "$PWD/jstests/third_party" -prune -o -name "*.js" -print \ find "jstests" "src/mongo/db/modules/enterprise" -path "$PWD/jstests/third_party" -prune -o -name "*.js" -print |
| xargs -P 32 -L 50 $CONTAINER_RUNTIME run -v $LOCAL_INPUT_DIR/jstests:$CONTAINER_INPUT_DIR/jstests -v $LOCAL_INPUT_DIR/src/mongo/db/modules/enterprise:$CONTAINER_INPUT_DIR/src/mongo/db/modules/enterprise 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest npm run-script parse-jsfiles -- 2>&1 \ xargs -P 32 -L 50 $CONTAINER_RUNTIME run -v $LOCAL_INPUT_DIR/jstests:$CONTAINER_INPUT_DIR/jstests -v $LOCAL_INPUT_DIR/src/mongo/db/modules/enterprise:$CONTAINER_INPUT_DIR/src/mongo/db/modules/enterprise 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest npm run-script parse-jsfiles -- 2>&1 |
| tee lint_fuzzer_sanity.log tee lint_fuzzer_sanity.log
exit_code=$? exit_code=$?
activate_venv activate_venv

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
set -o errexit set -o errexit

View File

@ -1,14 +0,0 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
set -o pipefail
cd src
PATH="/opt/shfmt/v3.2.4/bin:$PATH"
./buildscripts/shellscripts-linters.sh | tee shellscripts.log
exit_code=$?
activate_venv
$python ./buildscripts/simple_report.py --test-name shfmt --log-file shellscripts.log --exit-code $exit_code
exit $exit_code

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
set -o pipefail set -o pipefail

View File

@ -1,9 +1,9 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src
client_logs=$(ls crud*.log fsm*.log 2> /dev/null) client_logs=$(ls crud*.log fsm*.log 2>/dev/null)
if [ ! -z "$client_logs" ]; then if [ ! -z "$client_logs" ]; then
${tar} czf client-logs.tgz $client_logs ${tar} czf client-logs.tgz $client_logs
fi fi

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
set -o errexit set -o errexit

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
set -o errexit set -o errexit

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
set -o errexit set -o errexit

View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh" . "$DIR/prelude.sh"
cd src cd src

Some files were not shown because too many files have changed in this diff Show More