mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-101034 Use rules_lint shfmt formatter (#38448) 

GitOrigin-RevId: e8ef1ba2000e12fa2cd5a115a9ceeab92332e938
This commit is contained in:
Zack Winter 2025-07-15 18:19:08 -07:00 committed by MongoDB Bot
parent 5ac113adb1
commit 7ccc14bf91
174 changed files with 2259 additions and 2387 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.editorconfig Normal file
View File

@ -0,0 +1,9 @@
root = true
[*.sh]
charset = utf-8
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
indent_style = space
indent_size = 4

1
.github/CODEOWNERS vendored
View File

@ -10,6 +10,7 @@ OWNERS.yml @10gen/server-root-ownership @svc-auto-approve-bot
.bazel* @10gen/devprod-build @svc-auto-approve-bot
.clang-format @10gen/server-programmability @svc-auto-approve-bot
.clang-tidy.in @10gen/server-programmability @svc-auto-approve-bot
/.editorconfig @10gen/devprod-build @svc-auto-approve-bot
.git* @10gen/devprod-build @svc-auto-approve-bot
.mypy.ini @10gen/devprod-build @10gen/devprod-correctness @svc-auto-approve-bot
.prettierignore @10gen/devprod-correctness @svc-auto-approve-bot

3
OWNERS.yml
View File

@ -18,6 +18,9 @@ filters:
- ".clang-tidy.in":
approvers:
- 10gen/server-programmability
- "/.editorconfig":
approvers:
- 10gen/devprod-build
- ".git*":
approvers:
- 10gen/devprod-build

6
bazel/format/BUILD.bazel
View File

@ -6,8 +6,6 @@ py_binary(
args = [
"--prettier",
"$(location //:prettier)",
"--shellscripts-linters",
"$(location //buildscripts:shellscripts_linters)",
"--rules-lint-format",
"$(location :rules_lint_format)",
"--rules-lint-format-check",
@ -17,7 +15,6 @@ py_binary(
":rules_lint_format",
":rules_lint_format.check",
"//:prettier",
"//buildscripts:shellscripts_linters",
"@shfmt",
],
env = {
@ -38,9 +35,8 @@ format_multirun(
graphql = "//:prettier",
html = "//:prettier",
markdown = "//:prettier",
shell = "@shfmt//:shfmt",
sql = "//:prettier",
starlark = "@buildifier_prebuilt//:buildifier",
visibility = ["//visibility:public"],
# TODO(SERVER-101034): Enable rules_lint shfmt after sh files are reformatted with .editorconfig
# shell = "@shfmt//:shfmt",
)

23
bazel/format/rules_lint_format_wrapper.py
View File

@ -82,21 +82,6 @@ def run_rules_lint(
return True
def run_shellscripts_linters(shellscripts_linters: pathlib.Path, check: bool) -> bool:
try:
command = [str(shellscripts_linters)]
if not check:
print("Running shellscripts formatter")
command.append("fix")
else:
print("Running shellscripts linter")
repo_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
subprocess.run(command, check=True, env=os.environ, cwd=repo_path)
except subprocess.CalledProcessError:
return False
return True
def run_prettier(
prettier: pathlib.Path, check: bool, files_to_format: Union[List[str], str] = "all"
) -> bool:
@ -169,12 +154,6 @@ def main() -> int:
parser.add_argument(
"--prettier", help="Set the path to prettier", required=True, type=pathlib.Path
)
parser.add_argument(
"--shellscripts-linters",
help="Set the path to shellscripts_linters",
required=True,
type=pathlib.Path,
)
parser.add_argument(
"--rules-lint-format",
help="Set the path to rules_lint's formatter",
@ -200,7 +179,6 @@ def main() -> int:
args = parser.parse_args()
prettier_path: pathlib.Path = args.prettier.resolve()
shellscripts_linters_path: pathlib.Path = args.shellscripts_linters.resolve()
os.chdir(default_dir)
@ -235,7 +213,6 @@ def main() -> int:
if run_rules_lint(
args.rules_lint_format, args.rules_lint_format_check, args.check, files_to_format
)
and run_shellscripts_linters(shellscripts_linters_path, args.check)
and run_prettier(prettier_path, args.check, files_to_format)
else 1
)

2
bazel/remote_execution_container/repin_dockerfiles.sh
View File

@ -234,7 +234,7 @@ generate_dockerfile() {
fi
mkdir -p "$output_dir"
cat << EOF > "$output_dir/dockerfile"
cat <<EOF >"$output_dir/dockerfile"
# DO NOT EDIT.
#
# This Dockerfile is generated by the 'repin_dockerfiles.sh' script. To repin

12
buildscripts/BUILD.bazel
View File

@ -111,18 +111,6 @@ py_binary(
],
)
sh_binary(
name = "shellscripts_linters",
srcs = ["shellscripts-linters.sh"],
data = [
"@shfmt",
],
env = {
"SHFMT_PATH": "$(rootpath @shfmt//:shfmt)",
},
visibility = ["//visibility:public"],
)
py_library(
name = "mongo_toolchain",
srcs = [

12
buildscripts/auto_install_db_contrib_tool.sh
View File

@ -24,8 +24,8 @@ if [[ -f "$HOME/.zshrc" ]]; then
rc_file="$HOME/.zshrc"
fi
if ! command -v db-contrib-tool &> /dev/null; then
if ! python3 -c "import sys; sys.exit(sys.version_info < (3, 7))" &> /dev/null; then
if ! command -v db-contrib-tool &>/dev/null; then
if ! python3 -c "import sys; sys.exit(sys.version_info < (3, 7))" &>/dev/null; then
actual_version=$(python3 -c 'import sys; print(sys.version)')
echo "You must have python3.7+ installed. Detected version $actual_version."
echo "To avoid unexpected issues, python3.7+ will not be automatically installed."
@ -38,12 +38,12 @@ if ! command -v db-contrib-tool &> /dev/null; then
exit 1
fi
if command -v pipx &> /dev/null; then
if command -v pipx &>/dev/null; then
echo "Found pipx: $(command -v pipx)."
echo "Using it to install 'db-contrib-tool'."
echo
pipx ensurepath &> /dev/null
pipx ensurepath &>/dev/null
if [[ -f "$rc_file" ]]; then
source "$rc_file"
fi
@ -51,7 +51,7 @@ if ! command -v db-contrib-tool &> /dev/null; then
pipx install db-contrib-tool --python $(command -v python3) --force
echo
else
if ! python3 -m pipx --version &> /dev/null; then
if ! python3 -m pipx --version &>/dev/null; then
echo "Couldn't find pipx. Installing it as python3 module:"
echo " $(command -v python3) -m pip install pipx"
echo
@ -64,7 +64,7 @@ if ! command -v db-contrib-tool &> /dev/null; then
echo
fi
python3 -m pipx ensurepath &> /dev/null
python3 -m pipx ensurepath &>/dev/null
if [[ -f "$rc_file" ]]; then
source "$rc_file"
fi

59
buildscripts/consolidate-repos-enterprise.sh
View File

@ -26,43 +26,36 @@ mkdir -p "$repodir/yum/redhat"
# to support different $releasever values in yum repo configurations
#
if [ ! -e "$repodir/yum/redhat/7Server" ]
then
if [ ! -e "$repodir/yum/redhat/7Server" ]; then
ln -s 7 "$repodir/yum/redhat/7Server"
fi
if [ ! -e "$repodir/yum/redhat/6Server" ]
then
if [ ! -e "$repodir/yum/redhat/6Server" ]; then
ln -s 6 "$repodir/yum/redhat/6Server"
fi
if [ ! -e "$repodir/yum/redhat/5Server" ]
then
if [ ! -e "$repodir/yum/redhat/5Server" ]; then
ln -s 5 "$repodir/yum/redhat/5Server"
fi
echo "Scanning and copying package files from $source_dir"
echo ". = skipping existing file, @ = copying file"
for package in $(find "$source_dir" -not \( -path "$repodir" -prune \) -and \( -name \*.rpm -o -name \*.deb -o -name Release \))
do
for package in $(find "$source_dir" -not \( -path "$repodir" -prune \) -and \( -name \*.rpm -o -name \*.deb -o -name Release \)); do
new_package_location="$repodir$(echo "$package" | sed 's/\/var\/www-enterprise\/[^\/]*//;')"
# skip if the directory structure looks weird
#
if echo "$new_package_location" | grep -q /repo/
then
if echo "$new_package_location" | grep -q /repo/; then
continue
fi
# skip if not enterprise package
#
if ! echo "$new_package_location" | grep -q enterprise
then
if ! echo "$new_package_location" | grep -q enterprise; then
continue
fi
# skip if it's already there
#
if [ -e "$new_package_location" -a "$(basename "$package")" != "Release" ]
then
if [ -e "$new_package_location" -a "$(basename "$package")" != "Release" ]; then
echo -n .
else
mkdir -p "$(dirname "$new_package_location")"
@ -74,33 +67,28 @@ echo
# packages are in place, now create metadata
#
for debian_dir in "$repodir"/apt/ubuntu "$repodir"/apt/debian
do
for debian_dir in "$repodir"/apt/ubuntu "$repodir"/apt/debian; do
cd "$debian_dir"
for section_dir in $(find dists -type d -name multiverse -o -name main)
do
for arch_dir in "$section_dir"/{binary-i386,binary-amd64}
do
for section_dir in $(find dists -type d -name multiverse -o -name main); do
for arch_dir in "$section_dir"/{binary-i386,binary-amd64}; do
echo "Generating Packages file under $debian_dir/$arch_dir"
if [ ! -d $arch_dir ]
then
if [ ! -d $arch_dir ]; then
mkdir $arch_dir
fi
dpkg-scanpackages --multiversion "$arch_dir" > "$arch_dir"/Packages
gzip -9c "$arch_dir"/Packages > "$arch_dir"/Packages.gz
dpkg-scanpackages --multiversion "$arch_dir" >"$arch_dir"/Packages
gzip -9c "$arch_dir"/Packages >"$arch_dir"/Packages.gz
done
done
for release_file in $(find "$debian_dir" -name Release)
do
for release_file in $(find "$debian_dir" -name Release); do
release_dir=$(dirname "$release_file")
echo "Generating Release file under $release_dir"
cd $release_dir
tempfile=$(mktemp /tmp/ReleaseXXXXXX)
tempfile2=$(mktemp /tmp/ReleaseXXXXXX)
mv Release $tempfile
head -7 $tempfile > $tempfile2
apt-ftparchive release . >> $tempfile2
head -7 $tempfile >$tempfile2
apt-ftparchive release . >>$tempfile2
cp $tempfile2 Release
chmod 644 Release
rm Release.gpg
@ -118,28 +106,23 @@ done
# /var/www-enterprise/repo.consolidated/apt/ubuntu/dists/precise/mongodb-enterprise/unstable -> 2.5
# /var/www-enterprise/repo.consolidated/apt/debian/dists/wheezy/mongodb-enterprise/unstable -> 2.5
#
for unstable_branch_dir in "$repodir"/yum/redhat/*/*/$unstable_branch "$repodir"/yum/amazon/*/*/$unstable_branch "$repodir"/apt/debian/dists/*/*/$unstable_branch "$repodir"/apt/ubuntu/dists/*/*/$unstable_branch "$repodir"/zypper/suse/*/*/$unstable_branch
do
for unstable_branch_dir in "$repodir"/yum/redhat/*/*/$unstable_branch "$repodir"/yum/amazon/*/*/$unstable_branch "$repodir"/apt/debian/dists/*/*/$unstable_branch "$repodir"/apt/ubuntu/dists/*/*/$unstable_branch "$repodir"/zypper/suse/*/*/$unstable_branch; do
full_unstable_path=$(dirname "$unstable_branch_dir")/unstable
if [ -e "$unstable_branch_dir" -a ! -e "$full_unstable_path" ]
then
if [ -e "$unstable_branch_dir" -a ! -e "$full_unstable_path" ]; then
echo "Linking unstable branch directory $unstable_branch_dir to $full_unstable_path"
ln -s $unstable_branch $full_unstable_path
fi
done
for stable_branch_dir in "$repodir"/yum/redhat/*/*/$stable_branch "$repodir"/yum/amazon/*/*/$stable_branch "$repodir"/apt/debian/dists/*/*/$stable_branch "$repodir"/apt/ubuntu/dists/*/*/$stable_branch "$repodir"/zypper/suse/*/*/$stable_branch
do
for stable_branch_dir in "$repodir"/yum/redhat/*/*/$stable_branch "$repodir"/yum/amazon/*/*/$stable_branch "$repodir"/apt/debian/dists/*/*/$stable_branch "$repodir"/apt/ubuntu/dists/*/*/$stable_branch "$repodir"/zypper/suse/*/*/$stable_branch; do
full_stable_path=$(dirname "$stable_branch_dir")/stable
if [ -e "$stable_branch_dir" -a ! -e "$full_stable_path" ]
then
if [ -e "$stable_branch_dir" -a ! -e "$full_stable_path" ]; then
echo "Linking stable branch directory $stable_branch_dir to $full_stable_path"
ln -s $stable_branch $full_stable_path
fi
done
for rpm_dir in $(find "$repodir"/yum/redhat "$repodir"/zypper/suse -type d -name x86_64 -o -name i386)
do
for rpm_dir in $(find "$repodir"/yum/redhat "$repodir"/zypper/suse -type d -name x86_64 -o -name i386); do
echo "Generating redhat repo metadata under $rpm_dir"
cd "$rpm_dir"
createrepo .

59
buildscripts/consolidate-repos.sh
View File

@ -26,43 +26,36 @@ mkdir -p "$repodir/yum/redhat"
# to support different $releasever values in yum repo configurations
#
if [ ! -e "$repodir/yum/redhat/6Server" ]
then
if [ ! -e "$repodir/yum/redhat/6Server" ]; then
ln -s 6 "$repodir/yum/redhat/6Server"
fi
if [ ! -e "$repodir/yum/redhat/7Server" ]
then
if [ ! -e "$repodir/yum/redhat/7Server" ]; then
ln -s 7 "$repodir/yum/redhat/7Server"
fi
if [ ! -e "$repodir/yum/redhat/5Server" ]
then
if [ ! -e "$repodir/yum/redhat/5Server" ]; then
ln -s 5 "$repodir/yum/redhat/5Server"
fi
echo "Scanning and copying package files from $source_dir"
echo ". = skipping existing file, @ = copying file"
for package in $(find "$source_dir" -not \( -path "$repodir" -prune \) -and \( -name \*.rpm -o -name \*.deb -o -name Release \))
do
for package in $(find "$source_dir" -not \( -path "$repodir" -prune \) -and \( -name \*.rpm -o -name \*.deb -o -name Release \)); do
new_package_location="$repodir$(echo "$package" | sed 's/\/var\/www-org\/[^\/]*//;')"
# skip if the directory structure looks weird
#
if echo "$new_package_location" | grep -q /repo/
then
if echo "$new_package_location" | grep -q /repo/; then
continue
fi
# skip if not community package
#
if ! echo "$new_package_location" | grep -q org
then
if ! echo "$new_package_location" | grep -q org; then
continue
fi
# skip if it's already there
#
if [ -e "$new_package_location" -a "$(basename "$package")" != "Release" ]
then
if [ -e "$new_package_location" -a "$(basename "$package")" != "Release" ]; then
echo -n .
else
mkdir -p "$(dirname "$new_package_location")"
@ -74,33 +67,28 @@ echo
# packages are in place, now create metadata
#
for debian_dir in "$repodir"/apt/ubuntu "$repodir"/apt/debian
do
for debian_dir in "$repodir"/apt/ubuntu "$repodir"/apt/debian; do
cd "$debian_dir"
for section_dir in $(find dists -type d -name multiverse -o -name main)
do
for arch_dir in "$section_dir"/{binary-i386,binary-amd64}
do
for section_dir in $(find dists -type d -name multiverse -o -name main); do
for arch_dir in "$section_dir"/{binary-i386,binary-amd64}; do
echo "Generating Packages file under $debian_dir/$arch_dir"
if [ ! -d $arch_dir ]
then
if [ ! -d $arch_dir ]; then
mkdir $arch_dir
fi
dpkg-scanpackages --multiversion "$arch_dir" > "$arch_dir"/Packages
gzip -9c "$arch_dir"/Packages > "$arch_dir"/Packages.gz
dpkg-scanpackages --multiversion "$arch_dir" >"$arch_dir"/Packages
gzip -9c "$arch_dir"/Packages >"$arch_dir"/Packages.gz
done
done
for release_file in $(find "$debian_dir" -name Release)
do
for release_file in $(find "$debian_dir" -name Release); do
release_dir=$(dirname "$release_file")
echo "Generating Release file under $release_dir"
cd $release_dir
tempfile=$(mktemp /tmp/ReleaseXXXXXX)
tempfile2=$(mktemp /tmp/ReleaseXXXXXX)
mv Release $tempfile
head -7 $tempfile > $tempfile2
apt-ftparchive release . >> $tempfile2
head -7 $tempfile >$tempfile2
apt-ftparchive release . >>$tempfile2
cp $tempfile2 Release
chmod 644 Release
rm Release.gpg
@ -118,28 +106,23 @@ done
# /var/www-org/repo.consolidated/apt/ubuntu/dists/precise/mongodb-org/unstable -> 2.5
# /var/www-org/repo.consolidated/apt/debian/dists/wheezy/mongodb-org/unstable -> 2.5
#
for unstable_branch_dir in "$repodir"/yum/redhat/*/*/$unstable_branch "$repodir"/yum/amazon/*/*/$unstable_branch "$repodir"/apt/debian/dists/*/*/$unstable_branch "$repodir"/apt/ubuntu/dists/*/*/$unstable_branch "$repodir"/zypper/suse/*/*/$unstable_branch
do
for unstable_branch_dir in "$repodir"/yum/redhat/*/*/$unstable_branch "$repodir"/yum/amazon/*/*/$unstable_branch "$repodir"/apt/debian/dists/*/*/$unstable_branch "$repodir"/apt/ubuntu/dists/*/*/$unstable_branch "$repodir"/zypper/suse/*/*/$unstable_branch; do
full_unstable_path=$(dirname "$unstable_branch_dir")/unstable
if [ -e "$unstable_branch_dir" -a ! -e "$full_unstable_path" ]
then
if [ -e "$unstable_branch_dir" -a ! -e "$full_unstable_path" ]; then
echo "Linking unstable branch directory $unstable_branch_dir to $full_unstable_path"
ln -s $unstable_branch $full_unstable_path
fi
done
for stable_branch_dir in "$repodir"/yum/redhat/*/*/$stable_branch "$repodir"/yum/amazon/*/*/$stable_branch "$repodir"/apt/debian/dists/*/*/$stable_branch "$repodir"/apt/ubuntu/dists/*/*/$stable_branch "$repodir"/zypper/suse/*/*/$stable_branch
do
for stable_branch_dir in "$repodir"/yum/redhat/*/*/$stable_branch "$repodir"/yum/amazon/*/*/$stable_branch "$repodir"/apt/debian/dists/*/*/$stable_branch "$repodir"/apt/ubuntu/dists/*/*/$stable_branch "$repodir"/zypper/suse/*/*/$stable_branch; do
full_stable_path=$(dirname "$stable_branch_dir")/stable
if [ -e "$stable_branch_dir" -a ! -e "$full_stable_path" ]
then
if [ -e "$stable_branch_dir" -a ! -e "$full_stable_path" ]; then
echo "Linking stable branch directory $stable_branch_dir to $full_stable_path"
ln -s $stable_branch $full_stable_path
fi
done
for rpm_dir in $(find "$repodir"/yum/redhat "$repodir"/yum/amazon "$repodir"/zypper/suse -type d -name x86_64 -o -name i386)
do
for rpm_dir in $(find "$repodir"/yum/redhat "$repodir"/yum/amazon "$repodir"/zypper/suse -type d -name x86_64 -o -name i386); do
echo "Generating redhat repo metadata under $rpm_dir"
cd "$rpm_dir"
createrepo .

22
buildscripts/mount_drives.sh
View File

@ -12,7 +12,7 @@ user_group=$USER:$(id -Gn $USER | cut -f1 -d ' ')
# _usage_: Provides usage infomation
function _usage_ {
cat << EOF
cat <<EOF
usage: $0 options
This script supports the following parameters for Windows & Linux platforms:
-d <deviceNames>, REQUIRED, Space separated list of devices to mount /data on,
@ -31,10 +31,8 @@ This script supports the following parameters for Windows & Linux platforms:
EOF
}
# Parse command line options
while getopts "d:l:o:r:t:u:?" option
do
while getopts "d:l:o:r:t:u:?" option; do
case $option in
d)
data_device_names=$OPTARG
@ -54,7 +52,7 @@ do
u)
user_group=$OPTARG
;;
\?|*)
\? | *)
_usage_
exit 0
;;
@ -72,8 +70,7 @@ function mount_drive {
# Determine how many devices were specified
local num_devices=0
for device_name in $device_names
do
for device_name in $device_names; do
local devices="$devices /dev/$device_name"
let num_devices=num_devices+1
done
@ -93,8 +90,7 @@ function mount_drive {
local drive=$device_names
local system_drive=c
while true;
do
while true; do
sleep $drive_poll_delay
echo "Looking for drive '$drive' to mount $root_dir"
if [ -d /cygdrive/$drive ]; then
@ -105,8 +101,7 @@ function mount_drive {
cmd.exe /c mklink /J $system_drive:\\$root_dir $drive:\\$root_dir
ln -s /cygdrive/$drive/$root_dir /$root_dir
setfacl -s user::rwx,group::rwx,other::rwx /cygdrive/$drive/$root_dir
for sub_dir in $sub_dirs
do
for sub_dir in $sub_dirs; do
mkdir -p /cygdrive/$drive/$root_dir/$sub_dir
done
chown -R $user_group /cygdrive/$system_drive/$root_dir
@ -144,7 +139,7 @@ function mount_drive {
/sbin/udevadm control --stop-exec-queue
yes | /sbin/mdadm --create $device_name --level=0 -c256 --raid-devices=$num_devices $devices
/sbin/udevadm control --start-exec-queue
/sbin/mdadm --detail --scan > /etc/mdadm.conf
/sbin/mdadm --detail --scan >/etc/mdadm.conf
/sbin/blockdev --setra 32 $device_name
else
device_name="/dev/$device_names"
@ -165,8 +160,7 @@ function mount_drive {
mkdir /$root_dir || true
chmod 777 /$root_dir
mount -t $fs_type "UUID=$device_uuid" /$root_dir
for sub_dir in $sub_dirs
do
for sub_dir in $sub_dirs; do
mkdir -p /$root_dir/$sub_dir
chmod 1777 /$root_dir/$sub_dir
done

4
buildscripts/poetry_sync.sh
View File

@ -40,7 +40,7 @@ while getopts p:fin opt; do
esac
done
run () {
run() {
echo "$@"
if [[ "${dry_run}" == 1 ]]; then
return
@ -86,7 +86,7 @@ if ! "${py3}" -m pip show poetry &>/dev/null; then
fi
# we'll need to use pip this time around
if (( need_poetry_install )); then
if ((need_poetry_install)); then
run "${py3}" -m pip install "${pip_opts[@]}" -r poetry_requirements.txt
fi

4
buildscripts/setup_engflow_creds.sh
View File

@ -32,6 +32,6 @@ else
mv engflow.key $HOME/.engflow/creds
chown $USER $HOME/.engflow/creds/engflow.crt $HOME/.engflow/creds/engflow.key
chmod 600 $HOME/.engflow/creds/engflow.crt $HOME/.engflow/creds/engflow.key
echo "common --tls_client_certificate=$HOME/.engflow/creds/engflow.crt" >> $HOME/.bazelrc
echo "common --tls_client_key=$HOME/.engflow/creds/engflow.key" >> $HOME/.bazelrc
echo "common --tls_client_certificate=$HOME/.engflow/creds/engflow.crt" >>$HOME/.bazelrc
echo "common --tls_client_key=$HOME/.engflow/creds/engflow.key" >>$HOME/.bazelrc
fi

41
buildscripts/shellscripts-linters.sh
View File

@ -1,41 +0,0 @@
#!/bin/bash
set +o errexit
shfmt=shfmt
if [ -n "$SHFMT_PATH" ]; then
shfmt=$(readlink $SHFMT_PATH)
fi
if [ -n "$BUILD_WORKSPACE_DIRECTORY" ]; then
cd $BUILD_WORKSPACE_DIRECTORY
fi
if ! command -v $shfmt &>/dev/null; then
echo "Could not find shfmt at $shfmt"
exit 1
fi
lint_dirs="evergreen"
if [ "$1" = "fix" ]; then
$shfmt -w -i 2 -bn -sr "$lint_dirs"
fi
output_file="shfmt_output.txt"
exit_code=0
$shfmt -d -i 2 -bn -sr "$lint_dirs" >"$output_file"
if [ -s "$output_file" ]; then
echo "ERROR: Found formatting errors in shell script files in directories: $lint_dirs"
echo ""
cat "$output_file"
echo ""
echo "To fix formatting errors run"
echo ""
echo " ./buildscripts/shellscripts-linters.sh fix"
echo ""
exit_code=1
fi
rm -rf "$output_file"
exit "$exit_code"

1
buildscripts/tests/sbom_linter/inputs/import_script_with_mongo_prefix_version.sh
View File

@ -8,4 +8,3 @@ set -vx
NAME=protobuf
VERSION="mongo/v4.25.0"

8
buildscripts/yamllinters.sh
View File

@ -5,14 +5,14 @@ cd "$BASEDIR/../"
yamllint -c etc/yamllint_config.yml buildscripts etc jstests
PATH="$PATH:$HOME" evergreen evaluate etc/evergreen.yml > etc/evaluated_evergreen.yml
PATH="$PATH:$HOME" evergreen evaluate etc/evergreen_nightly.yml > etc/evaluated_evergreen_nightly.yml
PATH="$PATH:$HOME" evergreen evaluate etc/evergreen.yml >etc/evaluated_evergreen.yml
PATH="$PATH:$HOME" evergreen evaluate etc/evergreen_nightly.yml >etc/evaluated_evergreen_nightly.yml
# Remove references to the DSI repo before evergreen evaluate.
# The DSI module references break 'evaluate', the system_perf config should
# parse without them, and we don't want changes to the DSI repository to
# break checking that the rest of the imports etc. work.
awk '/lint_yaml trim start/{drop=1} /lint_yaml trim end/{drop=0} !drop' etc/system_perf.yml > etc/trimmed_system_perf.yml
PATH="$PATH:$HOME" evergreen evaluate etc/trimmed_system_perf.yml > etc/evaluated_system_perf.yml
awk '/lint_yaml trim start/{drop=1} /lint_yaml trim end/{drop=0} !drop' etc/system_perf.yml >etc/trimmed_system_perf.yml
PATH="$PATH:$HOME" evergreen evaluate etc/trimmed_system_perf.yml >etc/evaluated_system_perf.yml
python -m evergreen_lint -c ./etc/evergreen_lint.yml lint

19
etc/set_up_workstation.sh
View File

@ -1,5 +1,5 @@
silent_grep() {
command grep -q > /dev/null 2>&1 "$@"
command grep -q "$@" >/dev/null 2>&1
}
idem_file_append() {
@ -19,10 +19,10 @@ idem_file_append() {
local end_marker="# END $2"
if ! silent_grep "^$start_marker" "$1"; then
{
echo -e "\n$start_marker";
echo -e "$3";
echo -e "$end_marker";
} >> "$1"
echo -e "\n$start_marker"
echo -e "$3"
echo -e "$end_marker"
} >>"$1"
fi
}
@ -30,7 +30,8 @@ setup_bash() {
# Bash profile should source .bashrc
echo "################################################################################"
echo "Setting up bash..."
local block=$(cat <<BLOCK
local block=$(
cat <<BLOCK
if [[ -f ~/.bashrc ]]; then
source ~/.bashrc
fi
@ -67,7 +68,7 @@ setup_poetry() {
echo "################################################################################"
echo "Installing 'poetry' command..."
export PATH="$PATH:$HOME/.local/bin"
if command -v poetry &> /dev/null; then
if command -v poetry &>/dev/null; then
echo "'poetry' command exists; skipping setup"
else
pipx install poetry --pip-args="-r $(pwd)/poetry_requirements.txt"
@ -78,7 +79,7 @@ setup_poetry() {
setup_pipx() {
echo "################################################################################"
echo "Installing 'pipx' command..."
if command -v pipx &> /dev/null; then
if command -v pipx &>/dev/null; then
echo "'pipx' command exists; skipping setup"
else
export PATH="$PATH:$HOME/.local/bin"
@ -112,7 +113,7 @@ setup_db_contrib_tool() {
echo "################################################################################"
echo "Installing 'db-contrib-tool' command..."
export PATH="$PATH:$HOME/.local/bin"
if command -v db-contrib-tool &> /dev/null; then
if command -v db-contrib-tool &>/dev/null; then
echo "'db-contrib-tool' command exists; skipping setup"
else
pipx install db-contrib-tool

5
evergreen/BUILD.bazel
View File

@ -296,11 +296,6 @@ sh_binary(
srcs = ["lint_fuzzer_sanity_patch.sh"],
)
sh_binary(
name = "lint_shellscripts",
srcs = ["lint_shellscripts.sh"],
)
sh_binary(
name = "lint_yaml",
srcs = ["lint_yaml.sh"],

6
evergreen/antithesis_image_build_and_push.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
set -o errexit
@ -43,7 +43,7 @@ echo "docker daemon.json: set data-root to /data/mci/docker"
sudo service docker start
# Login
echo "${antithesis_repo_key}" > mongodb.key.json
echo "${antithesis_repo_key}" >mongodb.key.json
cat mongodb.key.json | sudo docker login -u _json_key https://us-central1-docker.pkg.dev --password-stdin
rm mongodb.key.json
@ -64,7 +64,7 @@ timeout -v 1800 docker exec workload buildscripts/resmoke.py run --suite ${suite
RET=$?
set -o errexit
docker-compose -f docker_compose/${suite}/docker-compose.yml logs > docker_logs.txt
docker-compose -f docker_compose/${suite}/docker-compose.yml logs >docker_logs.txt
docker-compose -f docker_compose/${suite}/docker-compose.yml down
# Change the permissions of all of the files in the docker compose directory to the current user.

2
evergreen/bazel.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
source "$DIR/bazel_utility_functions.sh"
(

6
evergreen/bazel_compile.sh
View File

@ -8,7 +8,7 @@
# * ${args} - List of additional Bazel arguments (e.g.: "--config=clang-tidy")
# Needed for evergreen scripts that use evergreen expansions and utility methods.
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -146,8 +146,8 @@ fi
for i in {1..3}; do
eval ${TIMEOUT_CMD} $BAZEL_BINARY build --verbose_failures $LOCAL_ARG ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \
--define=MONGO_VERSION=${version} ${patch_compile_flags} ${targets} 2>&1 | tee bazel_stdout.log \
&& RET=0 && break || RET=$? && sleep 60
--define=MONGO_VERSION=${version} ${patch_compile_flags} ${targets} 2>&1 | tee bazel_stdout.log &&
RET=0 && break || RET=$? && sleep 60
if [ $RET -eq 124 ]; then
echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..."
else

6
evergreen/bazel_coverage.sh
View File

@ -8,7 +8,7 @@
# * ${args} - Extra command line args to pass to "bazel coverage"
# Needed for evergreen scripts that use evergreen expansions and utility methods.
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -24,7 +24,7 @@ eval echo "Execution environment: Args: ${args} Target: ${target}"
BAZEL_BINARY=bazel
# Print command being run to file that can be uploaded
echo "python buildscripts/install_bazel.py" > bazel-invocation.txt
echo "python buildscripts/install_bazel.py" >bazel-invocation.txt
echo " bazel coverage ${args} ${target}" >> bazel-invocation.txt
echo " bazel coverage ${args} ${target}" >>bazel-invocation.txt
$BAZEL_BINARY coverage ${args} ${target}

12
evergreen/bazel_run.sh
View File

@ -8,7 +8,7 @@
# * ${redact_args} - If set, redact the args in the report
# Needed for evergreen scripts that use evergreen expansions and utility methods.
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -38,7 +38,7 @@ BAZEL_BINARY=$(bazel_get_binary_path)
# AL2 stores certs in a nonstandard location
if [[ -f /etc/os-release ]]; then
DISTRO=$(awk -F '[="]*' '/^PRETTY_NAME/ { print $2 }' < /etc/os-release)
DISTRO=$(awk -F '[="]*' '/^PRETTY_NAME/ { print $2 }' </etc/os-release)
if [[ $DISTRO == "Amazon Linux 2" ]]; then
export SSL_CERT_DIR=/etc/pki/tls/certs
export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
@ -55,14 +55,14 @@ else
fi
# Print command being run to file that can be uploaded
echo "python buildscripts/install_bazel.py" > bazel-invocation.txt
echo "bazel run --verbose_failures ${bazel_compile_flags} ${task_compile_flags} ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >> bazel-invocation.txt
echo "python buildscripts/install_bazel.py" >bazel-invocation.txt
echo "bazel run --verbose_failures ${bazel_compile_flags} ${task_compile_flags} ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >>bazel-invocation.txt
# Run bazel command, retrying up to five times
MAX_ATTEMPTS=5
for ((i = 1; i <= $MAX_ATTEMPTS; i++)); do
eval $env $BAZEL_BINARY run --verbose_failures $LOCAL_ARG ${target} ${args} >> bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 10
if [ $i -lt $MAX_ATTEMPTS ]; then echo "Bazel failed to execute, retrying ($(($i + 1)) of $MAX_ATTEMPTS attempts)... " >> bazel_output.log 2>&1; fi
eval $env $BAZEL_BINARY run --verbose_failures $LOCAL_ARG ${target} ${args} >>bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 10
if [ $i -lt $MAX_ATTEMPTS ]; then echo "Bazel failed to execute, retrying ($(($i + 1)) of $MAX_ATTEMPTS attempts)... " >>bazel_output.log 2>&1; fi
$BAZEL_BINARY shutdown
done

8
evergreen/bazel_test.sh
View File

@ -5,7 +5,7 @@
# * ${targets} - Test targets
# * ${bazel_args} - Extra command line args to pass to "bazel test"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -64,7 +64,7 @@ if [ -n "${test_timeout_sec}" ]; then
fi
ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} --define=MONGO_VERSION=${version} ${patch_compile_flags}"
echo ${ALL_FLAGS} > .bazel_build_flags
echo ${ALL_FLAGS} >.bazel_build_flags
set +o errexit
@ -80,8 +80,8 @@ for i in {1..3}; do
done
for i in {1..3}; do
eval ${TIMEOUT_CMD} ${BAZEL_BINARY} test ${ALL_FLAGS} ${targets} 2>&1 | tee bazel_stdout.log \
&& RET=0 && break || RET=$? && sleep 1
eval ${TIMEOUT_CMD} ${BAZEL_BINARY} test ${ALL_FLAGS} ${targets} 2>&1 | tee bazel_stdout.log &&
RET=0 && break || RET=$? && sleep 1
if [ $RET -eq 124 ]; then
echo "Bazel timed out after ${build_timeout_seconds} seconds, retrying..."
else

2
evergreen/check_idl_compat.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

2
evergreen/check_legacy_command_types.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

8
evergreen/check_workstation_setup.sh
View File

@ -2,21 +2,21 @@ failed_setup=false
source ~/.bashrc
if command -v pipx &> /dev/null; then
if command -v pipx &>/dev/null; then
echo "'pipx' command exists"
else
echo "pipx command not found - failed setup"
failed_setup=true
fi
if command -v poetry &> /dev/null; then
if command -v poetry &>/dev/null; then
echo "'poetry' command exists"
else
echo "poetry command not found - failed setup"
failed_setup=true
fi
if command -v db-contrib-tool &> /dev/null; then
if command -v db-contrib-tool &>/dev/null; then
echo "'db-contrib-tool' command exists"
else
echo "db-contrib-tool command not found - failed setup"
@ -26,7 +26,7 @@ fi
if test -d "./python3-venv"; then
echo "Venv directory exists, checking activation"
. python3-venv/bin/activate
./buildscripts/resmoke.py run --help &> /dev/null
./buildscripts/resmoke.py run --help &>/dev/null
if [ $? -eq 0 ]; then
echo "Virtual workstation set up correctly"
else

2
evergreen/compile_ninja.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

2
evergreen/compiled_binaries_get.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

4
evergreen/consolidate_benchmark_txt.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -10,5 +10,5 @@ OUTPUT_FILE="build/benchmarks.txt"
# Concatenate all text files in the directory into the output file
for file in build/*_bm.txt; do
cat "$file" >> "$OUTPUT_FILE"
cat "$file" >>"$OUTPUT_FILE"
done

2
evergreen/container_registry_login.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

2
evergreen/coverity_build.sh
View File

@ -10,7 +10,7 @@ python buildscripts/install_bazel.py
bazel_bin="$HOME/.local/bin/bazelisk"
# number of parallel jobs to use for build.
# Even with scale=0 (the default), bc command adds decimal digits in case of multiplication. Division by 1 gives us a whole number with scale=0
bazel_jobs=$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) * .85 / 1")
bazel_jobs=$(bc <<<"$(grep -c '^processor' /proc/cpuinfo) * .85 / 1")
build_config="--config=local --jobs=$bazel_jobs --compiler_type=gcc --opt=off --dbg=False --allocator=system"
bazel_query='mnemonic("CppCompile|LinkCompile", filter(//src/mongo, deps(//:install-core)) except //src/mongo/db/modules/enterprise/src/streams/third_party/...)'
bazel_cache="--output_user_root=$workdir/bazel_cache"

2
evergreen/crypt_run_tests.sh
View File

@ -3,7 +3,7 @@
# This script verifies that specific symbols, and specific symbols only are
# exported in mongo_crypt_v1.so
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

2
evergreen/do_jepsen_setup/create_fuzz_config.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
cd src

4
evergreen/do_jepsen_setup/nodes.sh
View File

@ -1,8 +1,8 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
cd src
set -o errexit
activate_venv
$python -c 'import socket; num_nodes = 5; print("\n".join(["%s:%d" % (socket.gethostname(), port) for port in range(20000, 20000 + num_nodes)]))' > nodes.txt
$python -c 'import socket; num_nodes = 5; print("\n".join(["%s:%d" % (socket.gethostname(), port) for port in range(20000, 20000 + num_nodes)]))' >nodes.txt

4
evergreen/external_auth_aws_setup.sh
View File

@ -1,10 +1,10 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
set -o errexit
cat << EOF > aws_e2e_setup.json
cat <<EOF >aws_e2e_setup.json
{
"iam_auth_ecs_account" : "${iam_auth_ecs_account}",
"iam_auth_ecs_secret_access_key" : "${iam_auth_ecs_secret_access_key}",

8
evergreen/external_auth_azure_setup.sh
View File

@ -1,10 +1,10 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
set -o errexit
cat << EOF > $HOME/azure_e2e_config.json
cat <<EOF >$HOME/azure_e2e_config.json
{
"tD548GwE1@outlook.com" : "${oidc_azure_test_user_account_one_secret}",
"tD548GwE2@outlook.com" : "${oidc_azure_test_user_account_two_secret}",
@ -24,12 +24,12 @@ cat << EOF > $HOME/azure_e2e_config.json
"oidc_azure_managed_identity_api_version": "${oidc_azure_managed_identity_api_version}"
}
EOF
cat << EOF > $HOME/oidc_azure_container_key
cat <<EOF >$HOME/oidc_azure_container_key
${oidc_azure_container_key}
EOF
# EVG project variables do not preserve line breaks so we store them as base64 and decode here
sed s/[[:space:]]//g $HOME/oidc_azure_container_key | base64 --decode > $HOME/azure_remote_key
sed s/[[:space:]]//g $HOME/oidc_azure_container_key | base64 --decode >$HOME/azure_remote_key
# Clean up temp file
rm -f $HOME/oidc_azure_container_key

2
evergreen/external_auth_azure_teardown.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
set -o errexit

8
evergreen/external_auth_gcp_setup.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -7,7 +7,7 @@ set -o errexit
# Create the config file, which will contain the GCE project/zone information along with
# the expected audience that will appear on the VM's ID token.
cat << EOF > $HOME/gce_vm_config.json
cat <<EOF >$HOME/gce_vm_config.json
{
"audience" : "${oidc_gcp_vm_id_token_audience}",
"projectID" : "${oidc_gcp_project_id}",
@ -19,7 +19,7 @@ EOF
# Create the SSH key file. Note that the SSH key has been base64 encoded and stored into an EVG
# environment variable, so it is first trimmed of any whitespace via sed and base64 decoded before
# being output to the file.
echo ${oidc_gcp_ssh_key} | sed "s/[[:space:]]//g" | base64 --decode > $HOME/gcp_ssh_key
echo ${oidc_gcp_ssh_key} | sed "s/[[:space:]]//g" | base64 --decode >$HOME/gcp_ssh_key
# Reduce SSH keyfile privileges so that it is secure enough for OpenSSH.
chmod 600 $HOME/gcp_ssh_key
@ -34,7 +34,7 @@ ls -al $HOME/gcp_ssh_key
# The contents of this file are expected to exist in base64 encoded format in
# $oidc_gcp_service_account_key, so the same steps are taken as above before dumping it into a
# newly-created JSON file.
echo ${oidc_gcp_service_account_key} | sed "s/[[:space:]]//g" | base64 --decode > ${GOOGLE_APPLICATION_CREDENTIALS}
echo ${oidc_gcp_service_account_key} | sed "s/[[:space:]]//g" | base64 --decode >${GOOGLE_APPLICATION_CREDENTIALS}
chmod 600 ${GOOGLE_APPLICATION_CREDENTIALS}
ls -al ${GOOGLE_APPLICATION_CREDENTIALS}

10
evergreen/external_auth_gcp_teardown.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -14,10 +14,10 @@ echo "Cleaning up OIDC GCP test artifacts"
# Delete the GCP VM specified in gce_vm_info.json if GOOGLE_APPLICATION_CREDENTIALS is set, points
# to a file, and the GCE config and VM info files exist.
if [ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" ] \
&& [ -f "${GOOGLE_APPLICATION_CREDENTIALS}" ] \
&& [ -f "${HOME}/gce_vm_config.json" ] \
&& [ -f "${HOME}/gce_vm_info.json" ]; then
if [ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" ] &&
[ -f "${GOOGLE_APPLICATION_CREDENTIALS}" ] &&
[ -f "${HOME}/gce_vm_config.json" ] &&
[ -f "${HOME}/gce_vm_info.json" ]; then
# Install google-cloud-compute so that the script can run.
$python -m pip install google-cloud-compute
$python src/mongo/db/modules/enterprise/jstests/external_auth_oidc_gcp/lib/gce_vm_manager.py delete --config_file $HOME/gce_vm_config.json --service_account_key_file ${GOOGLE_APPLICATION_CREDENTIALS} --output_file $HOME/gce_vm_info.json

4
evergreen/external_auth_oidc_setup.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -6,7 +6,7 @@ cd src
set -o errexit
# Should output contents to new file in home directory.
cat << EOF > $HOME/oidc_e2e_setup.json
cat <<EOF >$HOME/oidc_e2e_setup.json
{
"testserversecurityone@ping-test.com" : "${oidc_ping_test_user_account_one_secret}",
"testserversecuritytwo@ping-test.com" : "${oidc_ping_test_user_account_two_secret}",

2
evergreen/external_auth_oidc_teardown.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
# Only run this script for the external_auth_oidc_azure task.

16
evergreen/failed_unittests_gather.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -6,9 +6,9 @@ cd src
set -eou pipefail
# Only run on unit test tasks so we don't target mongod binaries from cores.
if [ "${task_name}" != "run_dbtest" ] \
&& [[ ${task_name} != integration_tests* ]] \
&& [[ "${task_name}" != unit_test_group*_no_sandbox ]]; then
if [ "${task_name}" != "run_dbtest" ] &&
[[ ${task_name} != integration_tests* ]] &&
[[ "${task_name}" != unit_test_group*_no_sandbox ]]; then
echo "Not gathering failed unittests binaries as this is not a unittest task: ${task_name}"
exit 0
fi
@ -17,14 +17,14 @@ unittest_bin_dir=dist-unittests/bin
mkdir -p $unittest_bin_dir || true
# Find all core files
core_files=$(/usr/bin/find -H . \( -name "dump_*.core" -o -name "*.mdmp" \) 2> /dev/null)
core_files=$(/usr/bin/find -H . \( -name "dump_*.core" -o -name "*.mdmp" \) 2>/dev/null)
while read -r core_file; do
# A core file name does not always have the executable name that generated it.
# See http://stackoverflow.com/questions/34801353/core-dump-filename-gets-thread-name-instead-of-executable-name-with-core-pattern
# On platforms with GDB, we get the binary name from core file
gdb=/opt/mongodbtoolchain/v5/bin/gdb
if [ -f $gdb ]; then
binary_file=$($gdb -batch --quiet -ex "core $core_file" 2> /dev/null | grep "Core was generated" | cut -f2 -d "\`" | cut -f1 -d "'" | cut -f1 -d " ")
binary_file=$($gdb -batch --quiet -ex "core $core_file" 2>/dev/null | grep "Core was generated" | cut -f2 -d "\`" | cut -f1 -d "'" | cut -f1 -d " ")
binary_file_locations=$binary_file
else
echo "Checking core file '$core_file'"
@ -34,7 +34,7 @@ while read -r core_file; do
# Locate the binary file. Since the base file name might be truncated, the find
# may return more than 1 file.
if [ "$binary_file" != "" ]; then
binary_file_locations=$(/usr/bin/find -H . -executable -name "$binary_file*${exe}" 2> /dev/null)
binary_file_locations=$(/usr/bin/find -H . -executable -name "$binary_file*${exe}" 2>/dev/null)
fi
fi
@ -84,7 +84,7 @@ while read -r core_file; do
fi
done
done <<< "${core_files}"
done <<<"${core_files}"
# Copy debug symbols for dynamic builds
lib_dir=bazel-bin/install/lib

2
evergreen/feature_flag_tags_check.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

4
evergreen/full_disk_debug.sh
View File

@ -14,11 +14,11 @@ while read -r output; do
echo "Running out of space \"$partition ($usep%)\" on $(hostname) as on $(date)"
HAS_FULL_DISK=true
fi
done <<< "$FILESYSTEMS"
done <<<"$FILESYSTEMS"
if $HAS_FULL_DISK; then
# print all files that are above one megabyte sorted
du -cha / 2> /dev/null | grep -E "^[0-9]+(\.[0-9]+)?[G|M|T]" | sort -h
du -cha / 2>/dev/null | grep -E "^[0-9]+(\.[0-9]+)?[G|M|T]" | sort -h
else
echo "No full partitions found, skipping"
fi

2
evergreen/functions/bolt.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
set +o errexit

4
evergreen/functions/credentials_setup.sh
View File

@ -1,9 +1,9 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
cd src
cat > mci.buildlogger << END_OF_CREDS
cat >mci.buildlogger <<END_OF_CREDS
slavename='${slave}'
passwd='${passwd}'
builder='${build_variant}_${project}'

2
evergreen/functions/download_pgo_profile.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
set +o errexit

4
evergreen/functions/evergreen_api_credentials_configure.sh
View File

@ -1,10 +1,10 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
cd src
# Create the Evergreen API credentials
cat > .evergreen.yml << END_OF_CREDS
cat >.evergreen.yml <<END_OF_CREDS
api_server_host: https://evergreen.mongodb.com/api
api_key: "${evergreen_api_key}"
user: "${evergreen_api_user}"

2
evergreen/functions/files_remove.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
if [ -z "${files}" ]; then

2
evergreen/functions/get_mongot_version.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
cd src

6
evergreen/functions/modified_patch_files_get_all.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
cd src
@ -9,11 +9,11 @@ set -o errexit
# For patch builds gather the modified patch files.
if [ "${is_patch}" = "true" ]; then
# Get list of patched files
git diff HEAD --name-only >> patch_files.txt
git diff HEAD --name-only >>patch_files.txt
if [ -d src/mongo/db/modules/enterprise ]; then
pushd src/mongo/db/modules/enterprise
# Update the patch_files.txt in the mongo repo.
git diff HEAD --name-only >> ~1/patch_files.txt
git diff HEAD --name-only >>~1/patch_files.txt
popd
fi
fi

6
evergreen/functions/notary_client_credentials_setup.sh
View File

@ -1,14 +1,14 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
cd src
set -o errexit
cat << EOF > notary_env.sh
cat <<EOF >notary_env.sh
export NOTARY_TOKEN=${signing_auth_token_70}
export BARQUE_USERNAME=${barque_user}
export BARQUE_API_KEY=${barque_api_key}
EOF
echo "${signing_auth_token_70}" > signing_auth_token
echo "${signing_auth_token_70}" >signing_auth_token

12
evergreen/functions/process_threads_monitor.sh
View File

@ -1,17 +1,17 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
proc_list="(java|lein|mongo|python|_test$|_test\.exe$)"
if [ "Windows_NT" = "$OS" ]; then
get_pids() {
proc_pids=$(tasklist /fo:csv \
| awk -F'","' '{x=$1; gsub("\"","",x); print $2, x}' \
| grep -iE $1 \
| cut -f1 -d ' ')
proc_pids=$(tasklist /fo:csv |
awk -F'","' '{x=$1; gsub("\"","",x); print $2, x}' |
grep -iE $1 |
cut -f1 -d ' ')
}
get_process_info() {
proc_name=""
proc_info=$(wmic process where "ProcessId=\"$1\"" get "Name,ProcessId,ThreadCount" /format:csv 2> /dev/null | grep $1)
proc_info=$(wmic process where "ProcessId=\"$1\"" get "Name,ProcessId,ThreadCount" /format:csv 2>/dev/null | grep $1)
if [ ! -z $proc_info ]; then
proc_name=$(echo $proc_info | cut -f2 -d ',')
proc_threads=$(echo $proc_info | cut -f4 -d ',')

10
evergreen/functions/remote_credentials_setup.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
# Since the macros 'private_key_remote' and 'private_key_file' are not always defined
@ -9,7 +9,7 @@ fi
if [ ! -z "${private_key_remote}" ] && [ ! -z "${private_key_file}" ]; then
mkdir -p ~/.ssh
private_key_file=$(eval echo "$private_key_file")
echo -n "${private_key_remote}" > ${private_key_file}
echo -n "${private_key_remote}" >${private_key_file}
chmod 0600 ${private_key_file}
fi
@ -28,19 +28,19 @@ if [ $aws_profile = "default" ]; then
else
aws_profile_config="[profile $aws_profile]"
fi
cat << EOF >> ~/.aws/config
cat <<EOF >>~/.aws/config
$aws_profile_config
region = us-east-1
EOF
# The profile in the credentials file is specified as [<profile>].
cat << EOF >> ~/.aws/credentials
cat <<EOF >>~/.aws/credentials
[$aws_profile]
aws_access_key_id = ${aws_key_remote}
aws_secret_access_key = ${aws_secret_remote}
EOF
cat << EOF > ~/.boto
cat <<EOF >~/.boto
[Boto]
https_validate_certificates = False
EOF

2
evergreen/functions/resmoke_jobs_determine.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
cd src

18
evergreen/functions/run_diskstats.sh
View File

@ -3,20 +3,20 @@ set -o verbose
# On Windows we can use typeperf.exe to dump performance counters.
if [ "Windows_NT" = "$OS" ]; then
typeperf -qx PhysicalDisk | grep Disk | grep -v _Total > disk_counters.txt
typeperf -qx PhysicalDisk | grep Disk | grep -v _Total >disk_counters.txt
typeperf -cf disk_counters.txt -si 5 -o mongo-diskstats
# Linux: iostat -t option for timestamp.
elif iostat -tdmx > /dev/null 2>&1; then
iostat -tdmx 5 > mongo-diskstats
elif iostat -tdmx >/dev/null 2>&1; then
iostat -tdmx 5 >mongo-diskstats
# OSX: Simulate the iostat timestamp.
elif iostat -d > /dev/null 2>&1; then
iostat -d -w 5 | while IFS= read -r line; do printf '%s %s\n' "$(date +'%m/%d/%Y %H:%M:%S')" "$line" >> mongo-diskstats; done
elif iostat -d >/dev/null 2>&1; then
iostat -d -w 5 | while IFS= read -r line; do printf '%s %s\n' "$(date +'%m/%d/%Y %H:%M:%S')" "$line" >>mongo-diskstats; done
# Check if vmstat -t is available.
elif vmstat -td > /dev/null 2>&1; then
vmstat -td 5 > mongo-diskstats
elif vmstat -td >/dev/null 2>&1; then
vmstat -td 5 >mongo-diskstats
# Check if vmstat -T d is available.
elif vmstat -T d > /dev/null 2>&1; then
vmstat -T d 5 > mongo-diskstats
elif vmstat -T d >/dev/null 2>&1; then
vmstat -T d 5 >mongo-diskstats
else
printf "Cannot collect mongo-diskstats on this platform\n"
fi

2
evergreen/functions/system_resource_info_collect.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
cd src

2
evergreen/functions/task_timeout_determine.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
cd src

2
evergreen/functions/tmp_directory_umount.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
set -o errexit

2
evergreen/functions/ulimit_info_collect.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
if [ "$(uname)" != "Linux" ] && [ "$(uname)" != "Darwin" ]; then

2
evergreen/functions/venv_adjust.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude_python.sh"
set -o errexit

10
evergreen/functions/venv_setup.sh
View File

@ -1,7 +1,7 @@
# exit immediately if virtualenv is not found
set -o errexit
evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)/.."
evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.."
. "$evergreen_dir/prelude_workdir.sh"
. "$evergreen_dir/prelude_python.sh"
@ -49,7 +49,7 @@ fi
"$python_loc" -m venv "$venv_dir"
# Adding README file for using this venv locally
cat << EOF >> venv_readme.txt
cat <<EOF >>venv_readme.txt
This is an archive of the Python venv generated by this Evergreen build.
You can use it locally to avoid needing to manually set up the Python environment.
@ -60,11 +60,11 @@ echo "Updating virtual env directory in activate script"
pushd venv; venv_dir=\$(pwd); popd
EOF
if [ "Windows_NT" = "$OS" ]; then
cat << EOF >> venv_readme.txt
cat <<EOF >>venv_readme.txt
sed -i -e "s:VIRTUAL_ENV=\".*\":VIRTUAL_ENV=\"\$venv_dir\":" "\$venv_dir/Scripts/activate"
EOF
else
cat << EOF >> venv_readme.txt
cat <<EOF >>venv_readme.txt
sed -i -e "s:VIRTUAL_ENV=\".*\":VIRTUAL_ENV=\"\$venv_dir\":" "\$venv_dir/bin/activate"
echo "Adding back python symlinks"
@ -133,4 +133,4 @@ fi
cd ..
python -m pip freeze > pip-requirements.txt
python -m pip freeze >pip-requirements.txt

2
evergreen/functions/version_expansions_generate.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
cd src

2
evergreen/functions/wiredtiger_develop_use.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
cd src

8
evergreen/garasign_gpg_crypt_sign.sh
View File

@ -1,10 +1,10 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_80}" >> "signing-envfile"
echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_80}" >> "signing-envfile"
echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_80}" >>"signing-envfile"
echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_80}" >>"signing-envfile"
set -o errexit
set -o verbose
@ -20,7 +20,7 @@ shasum -a 256 $crypt_file_name | tee $crypt_file_name.sha256
md5sum $crypt_file_name | tee $crypt_file_name.md5
# signing crypt linux artifact with gpg
cat << EOF >> gpg_signing_commands.sh
cat <<EOF >>gpg_signing_commands.sh
gpgloader # loading gpg keys.
gpg --yes -v --armor -o $crypt_file_name.sig --detach-sign $crypt_file_name
EOF

10
evergreen/garasign_gpg_sign.sh
View File

@ -1,10 +1,10 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_80}" >> "signing-envfile"
echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_80}" >> "signing-envfile"
echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_80}" >>"signing-envfile"
echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_80}" >>"signing-envfile"
set -o errexit
set -o verbose
@ -37,7 +37,7 @@ gen_checksums mongodb-src-$src_suffix.$long_ext
gen_checksums mongodb-cryptd-$push_name-$push_arch-$suffix.$ext
# signing linux artifacts with gpg
cat << 'EOF' > gpg_signing_commands.sh
cat <<'EOF' >gpg_signing_commands.sh
gpgloader # loading gpg keys.
function sign(){
if [ -e $1 ]
@ -50,7 +50,7 @@ function sign(){
EOF
cat << EOF >> gpg_signing_commands.sh
cat <<EOF >>gpg_signing_commands.sh
sign mongodb-$push_name-$push_arch-$suffix.$ext
sign mongodb-$push_name-$push_arch-debugsymbols-$suffix.$ext
sign mongodb-src-$src_suffix.$long_ext

10
evergreen/garasign_jsign_sign.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
if [ "${push_name}" != "windows" ]; then
@ -7,8 +7,8 @@ fi
cd src
echo "GRS_CONFIG_USER1_USERNAME=${garasign_jsign_username}" >> "signing-envfile"
echo "GRS_CONFIG_USER1_PASSWORD=${garasign_jsign_password}" >> "signing-envfile"
echo "GRS_CONFIG_USER1_USERNAME=${garasign_jsign_username}" >>"signing-envfile"
echo "GRS_CONFIG_USER1_PASSWORD=${garasign_jsign_password}" >>"signing-envfile"
set -o errexit
set -o verbose
@ -18,7 +18,7 @@ cp bazel-bin/src/mongo/installer/msi/mongodb-win32-x86_64-windows-${version}.msi
if [ "${is_patch}" != "true" ]; then
# signing windows artifacts with jsign
cat << 'EOF' > jsign_signing_commands.sh
cat <<'EOF' >jsign_signing_commands.sh
function sign(){
if [ -e $1 ]
then
@ -28,7 +28,7 @@ function sign(){
fi
}
EOF
cat << EOF >> jsign_signing_commands.sh
cat <<EOF >>jsign_signing_commands.sh
sign $msi_filename
EOF

2
evergreen/gather_mongo_coredumps.sh
View File

@ -1,6 +1,6 @@
cd src
# Find all core files and move to src
core_files=$(/usr/bin/find -H .. \( -name "*.core" -o -name "*.mdmp" \) 2> /dev/null)
core_files=$(/usr/bin/find -H .. \( -name "*.core" -o -name "*.mdmp" \) 2>/dev/null)
for core_file in $core_files; do
base_name=$(echo $core_file | sed "s/.*\///")
# Move file if it does not already exist

2
evergreen/gen_feature_flags.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

6
evergreen/gen_supplementary_data.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -59,11 +59,11 @@ if [ -z "${build_patch_id}" ] || [ -z "${reuse_compile_from}" ] || [ "${is_patch
uarch=$(uname -p)
os=$(uname -r)
json="{ \"version\": \"${version}\", \"gitVersion\": \"${revision}\", \"uarch\": \"$uarch\", \"os\": \"$os\" }"
echo $json | jq '.' > mongodb/atlas_info.json
echo $json | jq '.' >mongodb/atlas_info.json
# Add custom run_validate_collections.js wrapper
mv jstests/hooks/run_validate_collections.js jstests/hooks/run_validate_collections.actual.js
cat << EOF > jstests/hooks/run_validate_collections.js
cat <<EOF >jstests/hooks/run_validate_collections.js
print("NOTE: run_validate_collections.js will skip the oplog!");
TestData = { skipValidationNamespaces: ['local.oplog.rs'] };
await import("jstests/hooks/run_validate_collections.actual.js");

2
evergreen/gen_tasks_activate.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

2
evergreen/generate_buildid_debug_symbols_mapping.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

2
evergreen/generate_custom_build_promotion_expansions.sh
View File

@ -20,7 +20,7 @@ promote_revision=$(cat task_data.json | jq -r ".revision")
artifact_address="https://internal-downloads.mongodb.com/server-custom-builds/${promote_project_id}/${promote_version_id}/${promote_build_variant}/mongo-${promote_build_id}.tgz"
cat << EOT > ./promote-expansions.yml
cat <<EOT >./promote-expansions.yml
promote_project_id: "$promote_project_id"
promote_version_id: "$promote_version_id"
promote_build_id: "$promote_build_id"

44
evergreen/generate_evergreen_bazelrc.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -15,42 +15,42 @@ if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
if [[ "$PWD" != "$(cat Z:/bazel_tmp/mci_path)" ]]; then
echo "Clearing bazel output root from previous task mci '$(cat Z:/bazel_tmp/mci_path)'"
rm -rf Z:/bazel_tmp/* || true
echo $PWD > Z:/bazel_tmp/mci_path
echo $PWD >Z:/bazel_tmp/mci_path
fi
# Z:/ path is necessary to avoid running into MSVC's file length limit,
# see https://jira.mongodb.org/browse/DEVPROD-11126
abs_path=$(cygpath -w "$TMPDIR" | tr '\\' '/')
echo "startup --output_user_root=Z:/bazel_tmp" > .bazelrc.evergreen
echo "common --action_env=TMP=Z:/bazel_tmp" >> .bazelrc.evergreen
echo "common --action_env=TEMP=Z:/bazel_tmp" >> .bazelrc.evergreen
echo "BAZELISK_HOME=${abs_path}/bazelisk_home" >> .bazeliskrc
echo "startup --output_user_root=Z:/bazel_tmp" >.bazelrc.evergreen
echo "common --action_env=TMP=Z:/bazel_tmp" >>.bazelrc.evergreen
echo "common --action_env=TEMP=Z:/bazel_tmp" >>.bazelrc.evergreen
echo "BAZELISK_HOME=${abs_path}/bazelisk_home" >>.bazeliskrc
# echo "common --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" >> .bazelrc.git
echo "common --define GIT_COMMIT_HASH=nogitversion" >> .bazelrc.git
echo "common --define GIT_COMMIT_HASH=nogitversion" >>.bazelrc.git
else
echo "startup --output_user_root=${TMPDIR}/bazel-output-root" > .bazelrc.evergreen
echo "BAZELISK_HOME=${TMPDIR}/bazelisk_home" >> .bazeliskrc
echo "common --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" >> .bazelrc.git
echo "startup --output_user_root=${TMPDIR}/bazel-output-root" >.bazelrc.evergreen
echo "BAZELISK_HOME=${TMPDIR}/bazelisk_home" >>.bazeliskrc
echo "common --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" >>.bazelrc.git
fi
if [[ "${evergreen_remote_exec}" != "on" ]]; then
# Temporarily disable remote exec and only use remote cache
echo "common --remote_executor=" >> .bazelrc.evergreen
echo "common --modify_execution_info=.*=+no-remote-exec" >> .bazelrc.evergreen
echo "common --jobs=auto" >> .bazelrc.evergreen
echo "common --remote_executor=" >>.bazelrc.evergreen
echo "common --modify_execution_info=.*=+no-remote-exec" >>.bazelrc.evergreen
echo "common --jobs=auto" >>.bazelrc.evergreen
fi
uri="https://spruce.mongodb.com/task/${task_id:?}?execution=${execution:?}"
echo "common --tls_client_certificate=./engflow.cert" >> .bazelrc.evergreen
echo "common --tls_client_key=./engflow.key" >> .bazelrc.evergreen
echo "common --bes_keywords=engflow:CiCdPipelineName=${build_variant:?}" >> .bazelrc.evergreen
echo "common --bes_keywords=engflow:CiCdJobName=${task_name:?}" >> .bazelrc.evergreen
echo "common --bes_keywords=engflow:CiCdUri=${uri:?}" >> .bazelrc.evergreen
echo "common --bes_keywords=evg:project=${project:?}" >> .bazelrc.evergreen
echo "common --remote_upload_local_results=True" >> .bazelrc.evergreen
echo "common --test_output=summary" >> .bazelrc.evergreen
echo "common --tls_client_certificate=./engflow.cert" >>.bazelrc.evergreen
echo "common --tls_client_key=./engflow.key" >>.bazelrc.evergreen
echo "common --bes_keywords=engflow:CiCdPipelineName=${build_variant:?}" >>.bazelrc.evergreen
echo "common --bes_keywords=engflow:CiCdJobName=${task_name:?}" >>.bazelrc.evergreen
echo "common --bes_keywords=engflow:CiCdUri=${uri:?}" >>.bazelrc.evergreen
echo "common --bes_keywords=evg:project=${project:?}" >>.bazelrc.evergreen
echo "common --remote_upload_local_results=True" >>.bazelrc.evergreen
echo "common --test_output=summary" >>.bazelrc.evergreen
# Disable remote execution in evergreen only since it runs on every PR, but we still
# want it to be fast on workstations
echo "coverage --config=no-remote-exec" >> .bazelrc.evergreen
echo "coverage --config=no-remote-exec" >>.bazelrc.evergreen

2
evergreen/generate_version.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

2
evergreen/generate_version_burn_in.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

4
evergreen/get_all_resmoke_suite_configs.sh
View File

@ -4,7 +4,7 @@
# Usage:
# bash get_all_resmoke_suite_configs.sh
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -21,4 +21,4 @@ BAZEL_BINARY=$(bazel_get_binary_path)
# f.path for f in target.files.to_list() -> the path to the config file, like bazel-out/k8-fastbuild/bin/buildscripts/resmokeconfig/core.yml
${BAZEL_BINARY} cquery ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \
--define=MONGO_VERSION=${version} ${patch_compile_flags} "kind(resmoke_config, //...)" \
--output=starlark --starlark:expr "': '.join([str(target.label).replace('@@','')] + [f.path for f in target.files.to_list()])" > resmoke_suite_configs.yml
--output=starlark --starlark:expr "': '.join([str(target.label).replace('@@','')] + [f.path for f in target.files.to_list()])" >resmoke_suite_configs.yml

2
evergreen/get_bin_and_fcv_versions.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

4
evergreen/get_resmoke_suite_config.sh
View File

@ -6,7 +6,7 @@
# Required environment variables:
# * ${suite} - Resmoke bazel target, like //buildscripts/resmokeconfig:core
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -18,4 +18,4 @@ source ./evergreen/bazel_utility_functions.sh
BAZEL_BINARY=$(bazel_get_binary_path)
echo "suite_config: $(${BAZEL_BINARY} cquery ${bazel_args} ${bazel_compile_flags} ${task_compile_flags} \
--define=MONGO_VERSION=${version} ${patch_compile_flags} ${suite}_config --output files)" > suite_config_expansion.yml
--define=MONGO_VERSION=${version} ${patch_compile_flags} ${suite}_config --output files)" >suite_config_expansion.yml

2
evergreen/hang_analyzer.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

2
evergreen/idl_tests_run.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

2
evergreen/jepsen_docker/docker-up.sh
View File

@ -1,4 +1,4 @@
set -euo pipefail
cd jepsen/docker
./bin/up -n 9 -d 2>&1 > docker.log
./bin/up -n 9 -d 2>&1 >docker.log

4
evergreen/jepsen_docker/list-append.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
# this file does not use set -euo pipefail because we determine test success or
@ -71,7 +71,7 @@ for i in {1..30}; do
done
# Merge all jepsen_test_${i}.log into a single file
cat src/jepsen-mongodb/jepsen_test_*.log > src/jepsen-mongodb/jepsen_${task_name}_${execution}.log
cat src/jepsen-mongodb/jepsen_test_*.log >src/jepsen-mongodb/jepsen_${task_name}_${execution}.log
end_time=$(date +%s)
elapsed_secs=$((end_time - start_time))

2
evergreen/jepsen_docker/setup_config_fuzzer.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/../prelude.sh"
cd src

4
evergreen/jepsen_report.sh
View File

@ -16,10 +16,10 @@ else
final_exit_code=1
else
# If the failure is due to setup, then this is considered a system failure.
echo $grep_exit_code > jepsen_system_failure_${task_name}_${execution}
echo $grep_exit_code >jepsen_system_failure_${task_name}_${execution}
exit 0
fi
fi
# Create report.json
echo "{\"failures\": $failures, \"results\": [{\"status\": $status, \"exit_code\": $final_exit_code, \"test_file\": \"${task_name}\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > ../report.json
echo "{\"failures\": $failures, \"results\": [{\"status\": $status, \"exit_code\": $final_exit_code, \"test_file\": \"${task_name}\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" >../report.json
exit $final_exit_code

2
evergreen/jepsen_test_fail.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src/jepsen-mongodb

6
evergreen/jepsen_test_run.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src/jepsen-mongodb
@ -36,8 +36,8 @@ lein run test --test ${jepsen_test_name} \
${jepsen_storage_engine} \
${jepsen_time_limit} \
${jepsen_write_concern} \
2>&1 \
| tee jepsen_${task_name}_${execution}.log
2>&1 |
tee jepsen_${task_name}_${execution}.log
end_time=$(date +%s)
elapsed_secs=$((end_time - start_time))
. ../evergreen/jepsen_report.sh

2
evergreen/jstestfuzz_minimize.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src/jstestfuzz

2
evergreen/jstestfuzz_run.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
set -evo pipefail

8
evergreen/jstestfuzz_setup.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -6,11 +6,11 @@ cd src
set -o errexit
set -o verbose
if command -v docker > /dev/null 2>&1; then
if command -v docker >/dev/null 2>&1; then
echo "Docker is installed, using docker"
CONTAINER_RUNTIME=docker
elif command -v podman > /dev/null 2>&1; then
elif command -v podman >/dev/null 2>&1; then
echo "Podman is installed, using podman"
CONTAINER_RUNTIME=podman
else
@ -18,6 +18,6 @@ else
exit 1
fi
echo "CONTAINER_RUNTIME: ${CONTAINER_RUNTIME}" >> expansions.yml
echo "CONTAINER_RUNTIME: ${CONTAINER_RUNTIME}" >>expansions.yml
aws ecr get-login-password --region us-east-1 | $CONTAINER_RUNTIME login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com

8
evergreen/lint_fuzzer_sanity_all.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
@ -13,9 +13,9 @@ CONTAINER_INPUT_DIR="/app"
podman pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest
# Run parse-jsfiles on 50 files at a time with 32 processes in parallel.
# Skip javascript files in third_party directory
find "jstests" "src/mongo/db/modules/enterprise" -path "$PWD/jstests/third_party" -prune -o -name "*.js" -print \
| xargs -P 32 -L 50 $CONTAINER_RUNTIME run -v $LOCAL_INPUT_DIR/jstests:$CONTAINER_INPUT_DIR/jstests -v $LOCAL_INPUT_DIR/src/mongo/db/modules/enterprise:$CONTAINER_INPUT_DIR/src/mongo/db/modules/enterprise 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest npm run-script parse-jsfiles -- 2>&1 \
| tee lint_fuzzer_sanity.log
find "jstests" "src/mongo/db/modules/enterprise" -path "$PWD/jstests/third_party" -prune -o -name "*.js" -print |
xargs -P 32 -L 50 $CONTAINER_RUNTIME run -v $LOCAL_INPUT_DIR/jstests:$CONTAINER_INPUT_DIR/jstests -v $LOCAL_INPUT_DIR/src/mongo/db/modules/enterprise:$CONTAINER_INPUT_DIR/src/mongo/db/modules/enterprise 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest npm run-script parse-jsfiles -- 2>&1 |
tee lint_fuzzer_sanity.log
exit_code=$?
activate_venv

2
evergreen/lint_fuzzer_sanity_patch.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
set -o errexit

14
evergreen/lint_shellscripts.sh
View File

@ -1,14 +0,0 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
set -o pipefail
cd src
PATH="/opt/shfmt/v3.2.4/bin:$PATH"
./buildscripts/shellscripts-linters.sh | tee shellscripts.log
exit_code=$?
activate_venv
$python ./buildscripts/simple_report.py --test-name shfmt --log-file shellscripts.log --exit-code $exit_code
exit $exit_code

2
evergreen/lint_yaml.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
set -o pipefail

4
evergreen/local_client_logs_tar.sh
View File

@ -1,9 +1,9 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
client_logs=$(ls crud*.log fsm*.log 2> /dev/null)
client_logs=$(ls crud*.log fsm*.log 2>/dev/null)
if [ ! -z "$client_logs" ]; then
${tar} czf client-logs.tgz $client_logs
fi

2
evergreen/modify_debug_symbols.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
set -o errexit

2
evergreen/monitor_build_status_run.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
set -o errexit

2
evergreen/move_npm_logs.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
set -o errexit

2
evergreen/multiversion_exclude_tags_generate.sh
View File

@ -1,4 +1,4 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src

Some files were not shown because too many files have changed in this diff Show More