mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-79305 [v6.0] Build a skip compile param

This commit is contained in:
Simon Eismann 2023-08-14 16:27:30 +00:00 committed by Evergreen Agent
parent ea49a40fd3
commit fb6fb182d1
2 changed files with 197 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
buildscripts/download_sys_perf_binaries.py Normal file
View File

@ -0,0 +1,103 @@
#!/usr/bin/env python3
"""Download the binaries from a previous sys-perf run."""
import argparse
import requests
BASE_URI = 'https://evergreen.mongodb.com/rest/v2/'
def _get_auth_headers(evergreen_api_user, evergreen_api_key):
return {
'Api-User': evergreen_api_user,
'Api-Key': evergreen_api_key,
}
def _get_build_id(build_variant_name, version_id, auth_headers):
url = BASE_URI + 'versions/' + version_id
response = requests.get(url, headers=auth_headers)
if response.status_code != 200:
raise ValueError('Invalid version_id:', version_id)
version_json = response.json()
build_variants = version_json['build_variants_status']
for build_variant in build_variants:
if build_variant['build_variant'] == build_variant_name:
return build_variant['build_id']
raise RuntimeError('The compile-variant ' + build_variant_name +
' does not exist for the build with version_id ' + version_id)
# All of our sys-perf compile variants have exactly one task,
# so we can safely select the first one from the build variants tasks.
def _get_task_id(build_id, auth_headers):
url = BASE_URI + 'builds/' + build_id
response = requests.get(url, headers=auth_headers)
if response.status_code != 200:
raise RuntimeError('Unexpected error when trying to reach' + url)
build_json = response.json()
task_list = build_json['tasks']
if len(task_list) != 1:
raise RuntimeError('Recieved unexpected tasklist:', task_list)
return task_list[0]
# The API used here always grabs the latest execution
def _get_binary_details(task_id, auth_headers):
url = BASE_URI + 'tasks/' + task_id
response = requests.get(url, headers=auth_headers)
if response.status_code != 200:
raise RuntimeError('Unexpected error when trying to reach' + url)
task_json = response.json()
if task_json['status'] != 'success':
raise RuntimeError('The task ' + task_id + ' did not run sucessfully')
# The binary will always be the first artifact, unless we make large changes to system_perf.yml
artifacts = task_json['artifacts']
if len(artifacts) > 0 and artifacts[0]['name'].startswith('mongo'):
return artifacts[0]
raise RuntimeError('Unexpected list of artifacts:' + artifacts)
def _get_binary_url(version_id, build_variant, evergreen_api_user, evergreen_api_key):
auth_headers = _get_auth_headers(evergreen_api_user, evergreen_api_key)
build_id = _get_build_id(build_variant, version_id, auth_headers)
task_id = _get_task_id(build_id, auth_headers)
binary_json = _get_binary_details(task_id, auth_headers)
return binary_json['url']
def _download_binary_file(url, save_path):
response = requests.get(url, stream=True)
if response.status_code == 200:
with open(save_path, 'wb') as file:
for chunk in response.iter_content(chunk_size=1024):
file.write(chunk)
else:
raise RuntimeError('Failed to download the file ' + url)
def _download_sys_perf_binaries(version_id, build_variant, evergreen_api_user, evergreen_api_key):
url = _get_binary_url(version_id, build_variant, evergreen_api_user, evergreen_api_key)
_download_binary_file(url, 'binary.tar.gz')
if __name__ == '__main__':
argParser = argparse.ArgumentParser()
argParser.add_argument("-v", "--version_id",
help="Evergreen version_id from which binaries will be downloaded")
argParser.add_argument("-b", "--build_variant",
help="Build variant for which binaries will be downloaded")
argParser.add_argument(
"-u", "--evergreen_api_user",
help="Evergreen API user, see https://spruce.mongodb.com/preferences/cli")
argParser.add_argument("-k", "--evergreen_api_key",
help="Evergreen API key, see https://spruce.mongodb.com/preferences/cli")
args = argParser.parse_args()
_download_sys_perf_binaries(args.version_id, args.build_variant, args.evergreen_api_user,
args.evergreen_api_key)

33
etc/system_perf.yml
View File

@ -6,6 +6,8 @@ stepback: false
parameters:
- key: patch_compile_flags
description: "Additional SCons flags to be applied during scons compile invocations in this patch"
- key: reuse_compile_from
description: "Version_id of the commit/patch to reuse the compile artifacts from, e.g., sys_perf_6.0_90c65f9cc8fc4e6664a5848230abaa9b3f3b02f7"
variables:
@ -282,6 +284,7 @@ functions:
params:
working_dir: src
script: |
if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
set -o errexit
set -o verbose
@ -302,6 +305,9 @@ functions:
# and when pushing artifacts up to S3.
IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
buildscripts/generate_version_expansions.py --out version_expansions.yml
else
touch version_expansions.yml
fi
- command: expansions.update
params:
file: src/version_expansions.yml
@ -309,6 +315,7 @@ functions:
params:
working_dir: src
script: |
if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
set -o errexit
set -o verbose
@ -317,6 +324,9 @@ functions:
SCONS_CACHE_MODE=${scons_cache_mode|} USE_SCONS_CACHE=${use_scons_cache|false} \
IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
else
touch compile_expansions.yml
fi
- command: expansions.update
params:
file: src/compile_expansions.yml
@ -324,6 +334,7 @@ functions:
params:
working_dir: src/mongo-tools/src/github.com/mongodb/mongo-tools
script: |
if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
set -o verbose
set -o errexit
@ -332,7 +343,7 @@ functions:
set -o igncr
fi;
# set_goenv provides set_goenv(), print_ldflags() and print_tags() used below
# set_goenv provides set_goenv()
. ./set_goenv.sh
GOROOT="" set_goenv || exit
go version
@ -342,23 +353,27 @@ functions:
build_tools="$build_tools mongoreplay"
fi
for i in $build_tools; do
go build -ldflags "$(print_ldflags)" ${args} -tags "$(print_tags ${tooltags})" -o "../../../../../mongodb/bin/$i${exe|}" $i/main/$i.go
go build -o "../../../../../mongodb/bin/$i${exe|}" $i/main/$i.go
"../../../../../mongodb/bin/$i${exe|}" --version
done
fi
- command: shell.exec
params:
working_dir: src
script: |
if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
set -o errexit
set -o verbose
source "${workdir}/compile_venv/bin/activate"
python ./buildscripts/idl/gen_all_feature_flag_list.py
mkdir -p mongodb/feature_flags
cp ./all_feature_flags.txt mongodb/feature_flags
fi
- command: shell.exec
params:
working_dir: src
script: |
if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
set -o errexit
set -o verbose
source "${workdir}/compile_venv/bin/activate"
@ -382,6 +397,19 @@ functions:
# Put all matching mongo debug from the build directory in an archive in the same location
# as the library archive (i.e. mongodb/bin).
tar czvf mongodb${compile_variant|}-debugsymbols.tar.gz $(find ./build/cached -name mongo\*.debug -type f) --xform 's:^.*/:mongodb/bin/:'
fi
- command: shell.exec
params:
working_dir: src
script: |
if [ -n "${reuse_compile_from}" ] && [ "${is_patch|false}" = "true" ]; then
set -o errexit
set -o verbose
source "${workdir}/compile_venv/bin/activate"
python buildscripts/download_sys_perf_binaries.py -v ${reuse_compile_from} -b ${build_variant} -u ${evergreen_api_user} -k ${evergreen_api_key}
mv binary.tar.gz mongodb${compile_variant|}.tar.gz
fi
- command: s3.put
params:
aws_key: ${aws_key}
@ -400,6 +428,7 @@ functions:
remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile_variant|}-${version_id}-debugsymbols.tar.gz
bucket: mciuploads
permissions: public-read
optional: true
content_type: ${content_type|application/x-gzip}
display_name: mongo-debugsymbols.tgz
###