mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-79305 [v6.0] Build a skip compile param

This commit is contained in:
Simon Eismann 2023-08-14 16:27:30 +00:00 committed by Evergreen Agent
parent ea49a40fd3
commit fb6fb182d1
2 changed files with 197 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
buildscripts/download_sys_perf_binaries.py Normal file
View File

@ -0,0 +1,103 @@
#!/usr/bin/env python3
"""Download the binaries from a previous sys-perf run."""
import argparse
import requests
BASE_URI = 'https://evergreen.mongodb.com/rest/v2/'
def _get_auth_headers(evergreen_api_user, evergreen_api_key):
return {
'Api-User': evergreen_api_user,
'Api-Key': evergreen_api_key,
}
def _get_build_id(build_variant_name, version_id, auth_headers):
url = BASE_URI + 'versions/' + version_id
response = requests.get(url, headers=auth_headers)
if response.status_code != 200:
raise ValueError('Invalid version_id:', version_id)
version_json = response.json()
build_variants = version_json['build_variants_status']
for build_variant in build_variants:
if build_variant['build_variant'] == build_variant_name:
return build_variant['build_id']
raise RuntimeError('The compile-variant ' + build_variant_name +
' does not exist for the build with version_id ' + version_id)
# All of our sys-perf compile variants have exactly one task,
# so we can safely select the first one from the build variants tasks.
def _get_task_id(build_id, auth_headers):
url = BASE_URI + 'builds/' + build_id
response = requests.get(url, headers=auth_headers)
if response.status_code != 200:
raise RuntimeError('Unexpected error when trying to reach' + url)
build_json = response.json()
task_list = build_json['tasks']
if len(task_list) != 1:
raise RuntimeError('Recieved unexpected tasklist:', task_list)
return task_list[0]
# The API used here always grabs the latest execution
def _get_binary_details(task_id, auth_headers):
url = BASE_URI + 'tasks/' + task_id
response = requests.get(url, headers=auth_headers)
if response.status_code != 200:
raise RuntimeError('Unexpected error when trying to reach' + url)
task_json = response.json()
if task_json['status'] != 'success':
raise RuntimeError('The task ' + task_id + ' did not run sucessfully')
# The binary will always be the first artifact, unless we make large changes to system_perf.yml
artifacts = task_json['artifacts']
if len(artifacts) > 0 and artifacts[0]['name'].startswith('mongo'):
return artifacts[0]
raise RuntimeError('Unexpected list of artifacts:' + artifacts)
def _get_binary_url(version_id, build_variant, evergreen_api_user, evergreen_api_key):
auth_headers = _get_auth_headers(evergreen_api_user, evergreen_api_key)
build_id = _get_build_id(build_variant, version_id, auth_headers)
task_id = _get_task_id(build_id, auth_headers)
binary_json = _get_binary_details(task_id, auth_headers)
return binary_json['url']
def _download_binary_file(url, save_path):
response = requests.get(url, stream=True)
if response.status_code == 200:
with open(save_path, 'wb') as file:
for chunk in response.iter_content(chunk_size=1024):
file.write(chunk)
else:
raise RuntimeError('Failed to download the file ' + url)
def _download_sys_perf_binaries(version_id, build_variant, evergreen_api_user, evergreen_api_key):
url = _get_binary_url(version_id, build_variant, evergreen_api_user, evergreen_api_key)
_download_binary_file(url, 'binary.tar.gz')
if __name__ == '__main__':
argParser = argparse.ArgumentParser()
argParser.add_argument("-v", "--version_id",
help="Evergreen version_id from which binaries will be downloaded")
argParser.add_argument("-b", "--build_variant",
help="Build variant for which binaries will be downloaded")
argParser.add_argument(
"-u", "--evergreen_api_user",
help="Evergreen API user, see https://spruce.mongodb.com/preferences/cli")
argParser.add_argument("-k", "--evergreen_api_key",
help="Evergreen API key, see https://spruce.mongodb.com/preferences/cli")
args = argParser.parse_args()
_download_sys_perf_binaries(args.version_id, args.build_variant, args.evergreen_api_user,
args.evergreen_api_key)

159
etc/system_perf.yml
View File

@ -6,6 +6,8 @@ stepback: false
parameters: parameters:
- key: patch_compile_flags - key: patch_compile_flags
description: "Additional SCons flags to be applied during scons compile invocations in this patch" description: "Additional SCons flags to be applied during scons compile invocations in this patch"
- key: reuse_compile_from
description: "Version_id of the commit/patch to reuse the compile artifacts from, e.g., sys_perf_6.0_90c65f9cc8fc4e6664a5848230abaa9b3f3b02f7"
variables: variables:
@ -282,26 +284,30 @@ functions:
params: params:
working_dir: src working_dir: src
script: | script: |
set -o errexit if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
set -o verbose set -o errexit
set -o verbose
source "${workdir}/compile_venv/bin/activate" source "${workdir}/compile_venv/bin/activate"
# We get the raw version string (r1.2.3-45-gabcdef) from git # We get the raw version string (r1.2.3-45-gabcdef) from git
# And append "-sys-perf" at the end # And append "-sys-perf" at the end
export MONGO_VERSION=$(git describe --abbrev=7) export MONGO_VERSION=$(git describe --abbrev=7)
MONGO_VERSION="$MONGO_VERSION-sys-perf" MONGO_VERSION="$MONGO_VERSION-sys-perf"
# If this is a patch build, we add the patch version id to the version string so we know # If this is a patch build, we add the patch version id to the version string so we know
# this build was a patch, and which evergreen task it came from # this build was a patch, and which evergreen task it came from
if [ "${is_patch|false}" = "true" ]; then if [ "${is_patch|false}" = "true" ]; then
MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
fi
# This script handles sanitizing the version string for use during SCons build
# and when pushing artifacts up to S3.
IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
buildscripts/generate_version_expansions.py --out version_expansions.yml
else
touch version_expansions.yml
fi fi
# This script handles sanitizing the version string for use during SCons build
# and when pushing artifacts up to S3.
IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
buildscripts/generate_version_expansions.py --out version_expansions.yml
- command: expansions.update - command: expansions.update
params: params:
file: src/version_expansions.yml file: src/version_expansions.yml
@ -309,14 +315,18 @@ functions:
params: params:
working_dir: src working_dir: src
script: | script: |
set -o errexit if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
set -o verbose set -o errexit
set -o verbose
# This script handles whether the SCons cache should be used # This script handles whether the SCons cache should be used
source "${workdir}/compile_venv/bin/activate" source "${workdir}/compile_venv/bin/activate"
SCONS_CACHE_MODE=${scons_cache_mode|} USE_SCONS_CACHE=${use_scons_cache|false} \ SCONS_CACHE_MODE=${scons_cache_mode|} USE_SCONS_CACHE=${use_scons_cache|false} \
IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \ IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
python buildscripts/generate_compile_expansions.py --out compile_expansions.yml python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
else
touch compile_expansions.yml
fi
- command: expansions.update - command: expansions.update
params: params:
file: src/compile_expansions.yml file: src/compile_expansions.yml
@ -324,64 +334,82 @@ functions:
params: params:
working_dir: src/mongo-tools/src/github.com/mongodb/mongo-tools working_dir: src/mongo-tools/src/github.com/mongodb/mongo-tools
script: | script: |
set -o verbose if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
set -o errexit set -o verbose
set -o errexit
# make sure newlines in the scripts are handled correctly by windows # make sure newlines in the scripts are handled correctly by windows
if [ "Windows_NT" = "$OS" ]; then if [ "Windows_NT" = "$OS" ]; then
set -o igncr set -o igncr
fi; fi;
# set_goenv provides set_goenv(), print_ldflags() and print_tags() used below # set_goenv provides set_goenv()
. ./set_goenv.sh . ./set_goenv.sh
GOROOT="" set_goenv || exit GOROOT="" set_goenv || exit
go version go version
build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop" build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop"
if [ "${build_mongoreplay}" = "true" ]; then if [ "${build_mongoreplay}" = "true" ]; then
build_tools="$build_tools mongoreplay" build_tools="$build_tools mongoreplay"
fi
for i in $build_tools; do
go build -o "../../../../../mongodb/bin/$i${exe|}" $i/main/$i.go
"../../../../../mongodb/bin/$i${exe|}" --version
done
fi fi
for i in $build_tools; do
go build -ldflags "$(print_ldflags)" ${args} -tags "$(print_tags ${tooltags})" -o "../../../../../mongodb/bin/$i${exe|}" $i/main/$i.go
"../../../../../mongodb/bin/$i${exe|}" --version
done
- command: shell.exec - command: shell.exec
params: params:
working_dir: src working_dir: src
script: | script: |
set -o errexit if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
set -o verbose set -o errexit
source "${workdir}/compile_venv/bin/activate" set -o verbose
python ./buildscripts/idl/gen_all_feature_flag_list.py source "${workdir}/compile_venv/bin/activate"
mkdir -p mongodb/feature_flags python ./buildscripts/idl/gen_all_feature_flag_list.py
cp ./all_feature_flags.txt mongodb/feature_flags mkdir -p mongodb/feature_flags
cp ./all_feature_flags.txt mongodb/feature_flags
fi
- command: shell.exec - command: shell.exec
params: params:
working_dir: src working_dir: src
script: | script: |
set -o errexit if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
set -o verbose set -o errexit
source "${workdir}/compile_venv/bin/activate" set -o verbose
python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} install-core install-jstestshell MONGO_VERSION=${version} DESTDIR=$(pwd)/mongodb ${patch_compile_flags|} source "${workdir}/compile_venv/bin/activate"
mkdir -p mongodb/jstests/hooks python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} install-core install-jstestshell MONGO_VERSION=${version} DESTDIR=$(pwd)/mongodb ${patch_compile_flags|}
if [ -d jstests/hooks ] mkdir -p mongodb/jstests/hooks
then if [ -d jstests/hooks ]
echo "Fetching JS test DB correctness checks from directory jstests" then
cp -a jstests/* mongodb/jstests echo "Fetching JS test DB correctness checks from directory jstests"
cp -a jstests/* mongodb/jstests
echo "Now adding our own special run_validate_collections.js wrapper" echo "Now adding our own special run_validate_collections.js wrapper"
mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js
cat << EOF > mongodb/jstests/hooks/run_validate_collections.js cat << EOF > mongodb/jstests/hooks/run_validate_collections.js
print("NOTE: run_validate_collections.js will skip the oplog!"); print("NOTE: run_validate_collections.js will skip the oplog!");
TestData = { skipValidationNamespaces: ['local.oplog.rs'] }; TestData = { skipValidationNamespaces: ['local.oplog.rs'] };
load('jstests/hooks/run_validate_collections.actual.js'); load('jstests/hooks/run_validate_collections.actual.js');
EOF EOF
fi
tar czf mongodb${compile_variant|}.tar.gz mongodb
# Put all matching mongo debug from the build directory in an archive in the same location
# as the library archive (i.e. mongodb/bin).
tar czvf mongodb${compile_variant|}-debugsymbols.tar.gz $(find ./build/cached -name mongo\*.debug -type f) --xform 's:^.*/:mongodb/bin/:'
fi
- command: shell.exec
params:
working_dir: src
script: |
if [ -n "${reuse_compile_from}" ] && [ "${is_patch|false}" = "true" ]; then
set -o errexit
set -o verbose
source "${workdir}/compile_venv/bin/activate"
python buildscripts/download_sys_perf_binaries.py -v ${reuse_compile_from} -b ${build_variant} -u ${evergreen_api_user} -k ${evergreen_api_key}
mv binary.tar.gz mongodb${compile_variant|}.tar.gz
fi fi
tar czf mongodb${compile_variant|}.tar.gz mongodb
# Put all matching mongo debug from the build directory in an archive in the same location
# as the library archive (i.e. mongodb/bin).
tar czvf mongodb${compile_variant|}-debugsymbols.tar.gz $(find ./build/cached -name mongo\*.debug -type f) --xform 's:^.*/:mongodb/bin/:'
- command: s3.put - command: s3.put
params: params:
aws_key: ${aws_key} aws_key: ${aws_key}
@ -400,6 +428,7 @@ functions:
remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile_variant|}-${version_id}-debugsymbols.tar.gz remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile_variant|}-${version_id}-debugsymbols.tar.gz
bucket: mciuploads bucket: mciuploads
permissions: public-read permissions: public-read
optional: true
content_type: ${content_type|application/x-gzip} content_type: ${content_type|application/x-gzip}
display_name: mongo-debugsymbols.tgz display_name: mongo-debugsymbols.tgz
### ###