mirror of https://github.com/mongodb/mongo
SERVER-79305 [v6.0] Build a skip compile param
This commit is contained in:
Simon Eismann
Evergreen Agent
parent
ea49a40fd3
commit
fb6fb182d1
|
|
@ -0,0 +1,103 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Download the binaries from a previous sys-perf run."""
|
||||
|
||||
import argparse
|
||||
import requests
|
||||
|
||||
BASE_URI = 'https://evergreen.mongodb.com/rest/v2/'
|
||||
|
||||
|
||||
def _get_auth_headers(evergreen_api_user, evergreen_api_key):
|
||||
return {
|
||||
'Api-User': evergreen_api_user,
|
||||
'Api-Key': evergreen_api_key,
|
||||
}
|
||||
|
||||
|
||||
def _get_build_id(build_variant_name, version_id, auth_headers):
|
||||
url = BASE_URI + 'versions/' + version_id
|
||||
response = requests.get(url, headers=auth_headers)
|
||||
if response.status_code != 200:
|
||||
raise ValueError('Invalid version_id:', version_id)
|
||||
|
||||
version_json = response.json()
|
||||
build_variants = version_json['build_variants_status']
|
||||
for build_variant in build_variants:
|
||||
if build_variant['build_variant'] == build_variant_name:
|
||||
return build_variant['build_id']
|
||||
|
||||
raise RuntimeError('The compile-variant ' + build_variant_name +
|
||||
' does not exist for the build with version_id ' + version_id)
|
||||
|
||||
|
||||
# All of our sys-perf compile variants have exactly one task,
|
||||
# so we can safely select the first one from the build variants tasks.
|
||||
def _get_task_id(build_id, auth_headers):
|
||||
url = BASE_URI + 'builds/' + build_id
|
||||
response = requests.get(url, headers=auth_headers)
|
||||
if response.status_code != 200:
|
||||
raise RuntimeError('Unexpected error when trying to reach' + url)
|
||||
|
||||
build_json = response.json()
|
||||
task_list = build_json['tasks']
|
||||
if len(task_list) != 1:
|
||||
raise RuntimeError('Recieved unexpected tasklist:', task_list)
|
||||
return task_list[0]
|
||||
|
||||
|
||||
# The API used here always grabs the latest execution
|
||||
def _get_binary_details(task_id, auth_headers):
|
||||
url = BASE_URI + 'tasks/' + task_id
|
||||
response = requests.get(url, headers=auth_headers)
|
||||
if response.status_code != 200:
|
||||
raise RuntimeError('Unexpected error when trying to reach' + url)
|
||||
|
||||
task_json = response.json()
|
||||
if task_json['status'] != 'success':
|
||||
raise RuntimeError('The task ' + task_id + ' did not run sucessfully')
|
||||
|
||||
# The binary will always be the first artifact, unless we make large changes to system_perf.yml
|
||||
artifacts = task_json['artifacts']
|
||||
if len(artifacts) > 0 and artifacts[0]['name'].startswith('mongo'):
|
||||
return artifacts[0]
|
||||
raise RuntimeError('Unexpected list of artifacts:' + artifacts)
|
||||
|
||||
|
||||
def _get_binary_url(version_id, build_variant, evergreen_api_user, evergreen_api_key):
|
||||
auth_headers = _get_auth_headers(evergreen_api_user, evergreen_api_key)
|
||||
build_id = _get_build_id(build_variant, version_id, auth_headers)
|
||||
task_id = _get_task_id(build_id, auth_headers)
|
||||
binary_json = _get_binary_details(task_id, auth_headers)
|
||||
return binary_json['url']
|
||||
|
||||
|
||||
def _download_binary_file(url, save_path):
|
||||
response = requests.get(url, stream=True)
|
||||
if response.status_code == 200:
|
||||
with open(save_path, 'wb') as file:
|
||||
for chunk in response.iter_content(chunk_size=1024):
|
||||
file.write(chunk)
|
||||
else:
|
||||
raise RuntimeError('Failed to download the file ' + url)
|
||||
|
||||
|
||||
def _download_sys_perf_binaries(version_id, build_variant, evergreen_api_user, evergreen_api_key):
|
||||
url = _get_binary_url(version_id, build_variant, evergreen_api_user, evergreen_api_key)
|
||||
_download_binary_file(url, 'binary.tar.gz')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
argParser = argparse.ArgumentParser()
|
||||
argParser.add_argument("-v", "--version_id",
|
||||
help="Evergreen version_id from which binaries will be downloaded")
|
||||
argParser.add_argument("-b", "--build_variant",
|
||||
help="Build variant for which binaries will be downloaded")
|
||||
argParser.add_argument(
|
||||
"-u", "--evergreen_api_user",
|
||||
help="Evergreen API user, see https://spruce.mongodb.com/preferences/cli")
|
||||
argParser.add_argument("-k", "--evergreen_api_key",
|
||||
help="Evergreen API key, see https://spruce.mongodb.com/preferences/cli")
|
||||
args = argParser.parse_args()
|
||||
|
||||
_download_sys_perf_binaries(args.version_id, args.build_variant, args.evergreen_api_user,
|
||||
args.evergreen_api_key)
|
||||
|
|
@ -6,6 +6,8 @@ stepback: false
|
|||
parameters:
|
||||
- key: patch_compile_flags
|
||||
description: "Additional SCons flags to be applied during scons compile invocations in this patch"
|
||||
- key: reuse_compile_from
|
||||
description: "Version_id of the commit/patch to reuse the compile artifacts from, e.g., sys_perf_6.0_90c65f9cc8fc4e6664a5848230abaa9b3f3b02f7"
|
||||
|
||||
|
||||
variables:
|
||||
|
|
@ -282,26 +284,30 @@ functions:
|
|||
params:
|
||||
working_dir: src
|
||||
script: |
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
|
||||
# We get the raw version string (r1.2.3-45-gabcdef) from git
|
||||
# And append "-sys-perf" at the end
|
||||
export MONGO_VERSION=$(git describe --abbrev=7)
|
||||
MONGO_VERSION="$MONGO_VERSION-sys-perf"
|
||||
# We get the raw version string (r1.2.3-45-gabcdef) from git
|
||||
# And append "-sys-perf" at the end
|
||||
export MONGO_VERSION=$(git describe --abbrev=7)
|
||||
MONGO_VERSION="$MONGO_VERSION-sys-perf"
|
||||
|
||||
# If this is a patch build, we add the patch version id to the version string so we know
|
||||
# this build was a patch, and which evergreen task it came from
|
||||
if [ "${is_patch|false}" = "true" ]; then
|
||||
MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
|
||||
# If this is a patch build, we add the patch version id to the version string so we know
|
||||
# this build was a patch, and which evergreen task it came from
|
||||
if [ "${is_patch|false}" = "true" ]; then
|
||||
MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
|
||||
fi
|
||||
|
||||
# This script handles sanitizing the version string for use during SCons build
|
||||
# and when pushing artifacts up to S3.
|
||||
IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
|
||||
buildscripts/generate_version_expansions.py --out version_expansions.yml
|
||||
else
|
||||
touch version_expansions.yml
|
||||
fi
|
||||
|
||||
# This script handles sanitizing the version string for use during SCons build
|
||||
# and when pushing artifacts up to S3.
|
||||
IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
|
||||
buildscripts/generate_version_expansions.py --out version_expansions.yml
|
||||
- command: expansions.update
|
||||
params:
|
||||
file: src/version_expansions.yml
|
||||
|
|
@ -309,14 +315,18 @@ functions:
|
|||
params:
|
||||
working_dir: src
|
||||
script: |
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
# This script handles whether the SCons cache should be used
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
SCONS_CACHE_MODE=${scons_cache_mode|} USE_SCONS_CACHE=${use_scons_cache|false} \
|
||||
IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
|
||||
python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
|
||||
# This script handles whether the SCons cache should be used
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
SCONS_CACHE_MODE=${scons_cache_mode|} USE_SCONS_CACHE=${use_scons_cache|false} \
|
||||
IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
|
||||
python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
|
||||
else
|
||||
touch compile_expansions.yml
|
||||
fi
|
||||
- command: expansions.update
|
||||
params:
|
||||
file: src/compile_expansions.yml
|
||||
|
|
@ -324,64 +334,82 @@ functions:
|
|||
params:
|
||||
working_dir: src/mongo-tools/src/github.com/mongodb/mongo-tools
|
||||
script: |
|
||||
set -o verbose
|
||||
set -o errexit
|
||||
if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
|
||||
set -o verbose
|
||||
set -o errexit
|
||||
|
||||
# make sure newlines in the scripts are handled correctly by windows
|
||||
if [ "Windows_NT" = "$OS" ]; then
|
||||
set -o igncr
|
||||
fi;
|
||||
# make sure newlines in the scripts are handled correctly by windows
|
||||
if [ "Windows_NT" = "$OS" ]; then
|
||||
set -o igncr
|
||||
fi;
|
||||
|
||||
# set_goenv provides set_goenv(), print_ldflags() and print_tags() used below
|
||||
. ./set_goenv.sh
|
||||
GOROOT="" set_goenv || exit
|
||||
go version
|
||||
# set_goenv provides set_goenv()
|
||||
. ./set_goenv.sh
|
||||
GOROOT="" set_goenv || exit
|
||||
go version
|
||||
|
||||
build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop"
|
||||
if [ "${build_mongoreplay}" = "true" ]; then
|
||||
build_tools="$build_tools mongoreplay"
|
||||
build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop"
|
||||
if [ "${build_mongoreplay}" = "true" ]; then
|
||||
build_tools="$build_tools mongoreplay"
|
||||
fi
|
||||
for i in $build_tools; do
|
||||
go build -o "../../../../../mongodb/bin/$i${exe|}" $i/main/$i.go
|
||||
"../../../../../mongodb/bin/$i${exe|}" --version
|
||||
done
|
||||
fi
|
||||
for i in $build_tools; do
|
||||
go build -ldflags "$(print_ldflags)" ${args} -tags "$(print_tags ${tooltags})" -o "../../../../../mongodb/bin/$i${exe|}" $i/main/$i.go
|
||||
"../../../../../mongodb/bin/$i${exe|}" --version
|
||||
done
|
||||
- command: shell.exec
|
||||
params:
|
||||
working_dir: src
|
||||
script: |
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
python ./buildscripts/idl/gen_all_feature_flag_list.py
|
||||
mkdir -p mongodb/feature_flags
|
||||
cp ./all_feature_flags.txt mongodb/feature_flags
|
||||
if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
python ./buildscripts/idl/gen_all_feature_flag_list.py
|
||||
mkdir -p mongodb/feature_flags
|
||||
cp ./all_feature_flags.txt mongodb/feature_flags
|
||||
fi
|
||||
- command: shell.exec
|
||||
params:
|
||||
working_dir: src
|
||||
script: |
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} install-core install-jstestshell MONGO_VERSION=${version} DESTDIR=$(pwd)/mongodb ${patch_compile_flags|}
|
||||
mkdir -p mongodb/jstests/hooks
|
||||
if [ -d jstests/hooks ]
|
||||
then
|
||||
echo "Fetching JS test DB correctness checks from directory jstests"
|
||||
cp -a jstests/* mongodb/jstests
|
||||
if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} install-core install-jstestshell MONGO_VERSION=${version} DESTDIR=$(pwd)/mongodb ${patch_compile_flags|}
|
||||
mkdir -p mongodb/jstests/hooks
|
||||
if [ -d jstests/hooks ]
|
||||
then
|
||||
echo "Fetching JS test DB correctness checks from directory jstests"
|
||||
cp -a jstests/* mongodb/jstests
|
||||
|
||||
echo "Now adding our own special run_validate_collections.js wrapper"
|
||||
mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js
|
||||
echo "Now adding our own special run_validate_collections.js wrapper"
|
||||
mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js
|
||||
|
||||
cat << EOF > mongodb/jstests/hooks/run_validate_collections.js
|
||||
print("NOTE: run_validate_collections.js will skip the oplog!");
|
||||
TestData = { skipValidationNamespaces: ['local.oplog.rs'] };
|
||||
load('jstests/hooks/run_validate_collections.actual.js');
|
||||
cat << EOF > mongodb/jstests/hooks/run_validate_collections.js
|
||||
print("NOTE: run_validate_collections.js will skip the oplog!");
|
||||
TestData = { skipValidationNamespaces: ['local.oplog.rs'] };
|
||||
load('jstests/hooks/run_validate_collections.actual.js');
|
||||
EOF
|
||||
fi
|
||||
tar czf mongodb${compile_variant|}.tar.gz mongodb
|
||||
# Put all matching mongo debug from the build directory in an archive in the same location
|
||||
# as the library archive (i.e. mongodb/bin).
|
||||
tar czvf mongodb${compile_variant|}-debugsymbols.tar.gz $(find ./build/cached -name mongo\*.debug -type f) --xform 's:^.*/:mongodb/bin/:'
|
||||
fi
|
||||
- command: shell.exec
|
||||
params:
|
||||
working_dir: src
|
||||
script: |
|
||||
if [ -n "${reuse_compile_from}" ] && [ "${is_patch|false}" = "true" ]; then
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
python buildscripts/download_sys_perf_binaries.py -v ${reuse_compile_from} -b ${build_variant} -u ${evergreen_api_user} -k ${evergreen_api_key}
|
||||
mv binary.tar.gz mongodb${compile_variant|}.tar.gz
|
||||
fi
|
||||
tar czf mongodb${compile_variant|}.tar.gz mongodb
|
||||
# Put all matching mongo debug from the build directory in an archive in the same location
|
||||
# as the library archive (i.e. mongodb/bin).
|
||||
tar czvf mongodb${compile_variant|}-debugsymbols.tar.gz $(find ./build/cached -name mongo\*.debug -type f) --xform 's:^.*/:mongodb/bin/:'
|
||||
- command: s3.put
|
||||
params:
|
||||
aws_key: ${aws_key}
|
||||
|
|
@ -400,6 +428,7 @@ functions:
|
|||
remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile_variant|}-${version_id}-debugsymbols.tar.gz
|
||||
bucket: mciuploads
|
||||
permissions: public-read
|
||||
optional: true
|
||||
content_type: ${content_type|application/x-gzip}
|
||||
display_name: mongo-debugsymbols.tgz
|
||||
###
|
||||
|
|
|
|||
Loading…
Reference in New Issue