mirror of https://github.com/mongodb/mongo
SERVER-63191 Generate build expansions early
(cherry picked from commit 2836afbf9a)
This commit is contained in:
parent
48741207ef
commit
6fdbde504f
|
|
@ -7,9 +7,7 @@ $ python generate_compile_expansions.py --out compile_expansions.yml
|
|||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import shlex
|
||||
import yaml
|
||||
|
|
@ -25,7 +23,6 @@ def generate_expansions():
|
|||
"""
|
||||
args = parse_args()
|
||||
expansions = {}
|
||||
expansions.update(generate_version_expansions())
|
||||
expansions.update(generate_scons_cache_expansions())
|
||||
|
||||
with open(args.out, "w") as out:
|
||||
|
|
@ -40,39 +37,6 @@ def parse_args():
|
|||
return parser.parse_args()
|
||||
|
||||
|
||||
def generate_version_expansions():
|
||||
"""Generate expansions from a version.json file if given, or $MONGO_VERSION."""
|
||||
expansions = {}
|
||||
|
||||
if os.path.exists(VERSION_JSON):
|
||||
with open(VERSION_JSON, "r") as fh:
|
||||
data = fh.read()
|
||||
version_data = json.loads(data)
|
||||
version_line = version_data['version']
|
||||
version_parts = match_verstr(version_line)
|
||||
if not version_parts:
|
||||
raise ValueError("Unable to parse version.json")
|
||||
else:
|
||||
if not os.getenv("MONGO_VERSION"):
|
||||
raise Exception("$MONGO_VERSION not set and no version.json provided")
|
||||
version_line = os.getenv("MONGO_VERSION").lstrip("r")
|
||||
version_parts = match_verstr(version_line)
|
||||
if not version_parts:
|
||||
raise ValueError("Unable to parse version from stdin and no version.json provided")
|
||||
|
||||
if version_parts[0]:
|
||||
expansions["suffix"] = "v6.0-latest"
|
||||
expansions["src_suffix"] = "v6.0-latest"
|
||||
expansions["is_release"] = "false"
|
||||
else:
|
||||
expansions["suffix"] = version_line
|
||||
expansions["src_suffix"] = "r{0}".format(version_line)
|
||||
expansions["is_release"] = "true"
|
||||
expansions["version"] = version_line
|
||||
|
||||
return expansions
|
||||
|
||||
|
||||
def generate_scons_cache_expansions():
|
||||
"""Generate scons cache expansions from some files and environment variables."""
|
||||
expansions = {}
|
||||
|
|
@ -101,21 +65,5 @@ def generate_scons_cache_expansions():
|
|||
return expansions
|
||||
|
||||
|
||||
def match_verstr(verstr):
|
||||
"""Match a version string and capture the "extra" part.
|
||||
|
||||
If the version is a release like "2.3.4" or "2.3.4-rc0", this will return
|
||||
None. If the version is a pre-release like "2.3.4-325-githash" or
|
||||
"2.3.4-pre-", this will return "-pre-" or "-325-githash" If the version
|
||||
begins with the letter 'r', it will also match, e.g. r2.3.4, r2.3.4-rc0,
|
||||
r2.3.4-git234, r2.3.4-rc0-234-githash If the version is invalid (i.e.
|
||||
doesn't start with "2.3.4" or "2.3.4-rc0", this will return False.
|
||||
"""
|
||||
res = re.match(r'^r?(?:\d+\.\d+\.\d+(?:-rc\d+|-alpha\d+)?)(-.*)?', verstr)
|
||||
if not res:
|
||||
return False
|
||||
return res.groups()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
generate_expansions()
|
||||
|
|
|
|||
|
|
@ -7,9 +7,7 @@ $ python generate_compile_expansions.py --out compile_expansions.yml
|
|||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import shlex
|
||||
import yaml
|
||||
|
|
@ -25,7 +23,6 @@ def generate_expansions():
|
|||
"""
|
||||
args = parse_args()
|
||||
expansions = {}
|
||||
expansions.update(generate_version_expansions())
|
||||
expansions.update(generate_scons_cache_expansions())
|
||||
|
||||
with open(args.out, "w") as out:
|
||||
|
|
@ -40,39 +37,6 @@ def parse_args():
|
|||
return parser.parse_args()
|
||||
|
||||
|
||||
def generate_version_expansions():
|
||||
"""Generate expansions from a version.json file if given, or $MONGO_VERSION."""
|
||||
expansions = {}
|
||||
|
||||
if os.path.exists(VERSION_JSON):
|
||||
with open(VERSION_JSON, "r") as fh:
|
||||
data = fh.read()
|
||||
version_data = json.loads(data)
|
||||
version_line = version_data['version']
|
||||
version_parts = match_verstr(version_line)
|
||||
if not version_parts:
|
||||
raise ValueError("Unable to parse version.json")
|
||||
else:
|
||||
if not os.getenv("MONGO_VERSION"):
|
||||
raise Exception("$MONGO_VERSION not set and no version.json provided")
|
||||
version_line = os.getenv("MONGO_VERSION").lstrip("r")
|
||||
version_parts = match_verstr(version_line)
|
||||
if not version_parts:
|
||||
raise ValueError("Unable to parse version from stdin and no version.json provided")
|
||||
|
||||
if version_parts[0]:
|
||||
expansions["suffix"] = "v6.0-latest"
|
||||
expansions["src_suffix"] = "v6.0-latest"
|
||||
expansions["is_release"] = "false"
|
||||
else:
|
||||
expansions["suffix"] = version_line
|
||||
expansions["src_suffix"] = "r{0}".format(version_line)
|
||||
expansions["is_release"] = "true"
|
||||
expansions["version"] = version_line
|
||||
|
||||
return expansions
|
||||
|
||||
|
||||
def generate_scons_cache_expansions():
|
||||
"""Generate scons cache expansions from some files and environment variables."""
|
||||
expansions = {}
|
||||
|
|
@ -125,21 +89,5 @@ def generate_scons_cache_expansions():
|
|||
return expansions
|
||||
|
||||
|
||||
def match_verstr(verstr):
|
||||
"""Match a version string and capture the "extra" part.
|
||||
|
||||
If the version is a release like "2.3.4" or "2.3.4-rc0", this will return
|
||||
None. If the version is a pre-release like "2.3.4-325-githash" or
|
||||
"2.3.4-pre-", this will return "-pre-" or "-325-githash" If the version
|
||||
begins with the letter 'r', it will also match, e.g. r2.3.4, r2.3.4-rc0,
|
||||
r2.3.4-git234, r2.3.4-rc0-234-githash If the version is invalid (i.e.
|
||||
doesn't start with "2.3.4" or "2.3.4-rc0", this will return False.
|
||||
"""
|
||||
res = re.match(r'^r?(?:\d+\.\d+\.\d+(?:-rc\d+|-alpha\d+)?)(-.*)?', verstr)
|
||||
if not res:
|
||||
return False
|
||||
return res.groups()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
generate_expansions()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,93 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate the version expansions file used by Evergreen as part of the push/release process.
|
||||
|
||||
Invoke by specifying an output file.
|
||||
$ python generate_build_expansions.py --out version_expansions.yml
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import yaml
|
||||
|
||||
VERSION_JSON = "version.json"
|
||||
|
||||
|
||||
def generate_expansions():
|
||||
"""Entry point for the script.
|
||||
|
||||
This calls functions to generate version and scons cache expansions and
|
||||
writes them to a file.
|
||||
"""
|
||||
args = parse_args()
|
||||
expansions = {}
|
||||
expansions.update(generate_version_expansions())
|
||||
|
||||
with open(args.out, "w") as out:
|
||||
print("saving compile expansions to {0}: ({1})".format(args.out, expansions))
|
||||
yaml.safe_dump(expansions, out, default_flow_style=False)
|
||||
|
||||
|
||||
def parse_args():
|
||||
"""Parse program arguments."""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--out", required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def generate_version_expansions():
|
||||
"""Generate expansions from a version.json file if given, or $MONGO_VERSION."""
|
||||
expansions = {}
|
||||
|
||||
if os.path.exists(VERSION_JSON):
|
||||
with open(VERSION_JSON, "r") as fh:
|
||||
data = fh.read()
|
||||
version_data = json.loads(data)
|
||||
version_line = version_data['version']
|
||||
version_parts = match_verstr(version_line)
|
||||
if not version_parts:
|
||||
raise ValueError("Unable to parse version.json")
|
||||
else:
|
||||
version_line = os.getenv("MONGO_VERSION")
|
||||
if not version_line:
|
||||
raise Exception("$MONGO_VERSION not set and no version.json provided")
|
||||
|
||||
version_line = version_line.lstrip("r")
|
||||
version_parts = match_verstr(version_line)
|
||||
if not version_parts:
|
||||
raise ValueError("Unable to parse version from stdin and no version.json provided")
|
||||
|
||||
if version_parts[0]:
|
||||
expansions["suffix"] = "latest"
|
||||
expansions["src_suffix"] = "latest"
|
||||
expansions["is_release"] = "false"
|
||||
else:
|
||||
expansions["suffix"] = version_line
|
||||
expansions["src_suffix"] = "r{0}".format(version_line)
|
||||
expansions["is_release"] = "true"
|
||||
expansions["version"] = version_line
|
||||
|
||||
return expansions
|
||||
|
||||
|
||||
def match_verstr(verstr):
|
||||
"""Match a version string and capture the "extra" part.
|
||||
|
||||
If the version is a release like "2.3.4" or "2.3.4-rc0", this will return
|
||||
None. If the version is a pre-release like "2.3.4-325-githash" or
|
||||
"2.3.4-pre-", this will return "-pre-" or "-325-githash" If the version
|
||||
begins with the letter 'r', it will also match, e.g. r2.3.4, r2.3.4-rc0,
|
||||
r2.3.4-git234, r2.3.4-rc0-234-githash If the version is invalid (i.e.
|
||||
doesn't start with "2.3.4" or "2.3.4-rc0", this will return False.
|
||||
"""
|
||||
res = re.match(r'^r?(?:\d+\.\d+\.\d+(?:-rc\d+|-alpha\d+)?)(-.*)?', verstr)
|
||||
if not res:
|
||||
return False
|
||||
return res.groups()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
generate_expansions()
|
||||
|
|
@ -510,6 +510,20 @@ functions:
|
|||
args:
|
||||
- "src/evergreen/functions/binaries_extract.sh"
|
||||
|
||||
"get version expansions": &get_version_expansions
|
||||
command: s3.get
|
||||
params:
|
||||
aws_key: ${aws_key}
|
||||
aws_secret: ${aws_secret}
|
||||
remote_file: ${project}/${version_id}/version_expansions.yml
|
||||
bucket: mciuploads
|
||||
local_file: src/version_expansions.yml
|
||||
|
||||
"apply version expansions": &apply_version_expansions
|
||||
command: expansions.update
|
||||
params:
|
||||
file: src/version_expansions.yml
|
||||
|
||||
"check binary version": &check_binary_version
|
||||
command: subprocess.exec
|
||||
params:
|
||||
|
|
@ -795,7 +809,7 @@ functions:
|
|||
remote_file: ${project}/${build_variant}/${revision}/pip-requirements-${task_id}-${execution}.txt
|
||||
bucket: mciuploads
|
||||
permissions: public-read
|
||||
content_type: atext-plain
|
||||
content_type: text/plain
|
||||
display_name: Pip Requirements
|
||||
|
||||
"send benchmark results":
|
||||
|
|
@ -846,6 +860,8 @@ functions:
|
|||
- *adjust_venv
|
||||
- *fetch_binaries
|
||||
- *extract_binaries
|
||||
- *apply_version_expansions
|
||||
- *f_expansions_write
|
||||
- *check_binary_version
|
||||
- *get_buildnumber
|
||||
- *f_expansions_write
|
||||
|
|
@ -1224,6 +1240,8 @@ functions:
|
|||
- "src/evergreen/scons_lint.sh"
|
||||
|
||||
"scons compile":
|
||||
- *get_version_expansions
|
||||
- *apply_version_expansions
|
||||
- *f_expansions_write
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
|
|
@ -1232,6 +1250,24 @@ functions:
|
|||
args:
|
||||
- "src/evergreen/scons_compile.sh"
|
||||
|
||||
"generate version expansions":
|
||||
- *f_expansions_write
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
binary: bash
|
||||
args:
|
||||
- "src/evergreen/functions/version_expansions_generate.sh"
|
||||
- command: s3.put
|
||||
params:
|
||||
aws_key: ${aws_key}
|
||||
aws_secret: ${aws_secret}
|
||||
local_file: src/version_expansions.yml
|
||||
remote_file: ${project}/${version_id}/version_expansions.yml
|
||||
bucket: mciuploads
|
||||
permissions: public-read
|
||||
content_type: application/x-yaml
|
||||
display_name: version expansions
|
||||
|
||||
"generate compile expansions":
|
||||
- *f_expansions_write
|
||||
- command: subprocess.exec
|
||||
|
|
@ -2076,7 +2112,7 @@ functions:
|
|||
remote_file: ${project}/${build_variant}/${revision}/local-resmoke-invocation-${task_id}-${execution}.txt
|
||||
bucket: mciuploads
|
||||
permissions: public-read
|
||||
content_type: atext-plain
|
||||
content_type: text/plain
|
||||
display_name: Resmoke.py Invocation for Local Usage
|
||||
|
||||
|
||||
|
|
@ -2125,7 +2161,9 @@ tasks:
|
|||
## compile - build all scons targets except unittests ##
|
||||
- name: compile_dist_test
|
||||
tags: []
|
||||
depends_on: []
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- func: "scons compile"
|
||||
vars:
|
||||
|
|
@ -2150,7 +2188,6 @@ tasks:
|
|||
depends_on:
|
||||
- name: compile_dist_test
|
||||
commands:
|
||||
- *f_expansions_write
|
||||
- func: "scons compile"
|
||||
vars:
|
||||
targets: >-
|
||||
|
|
@ -2214,6 +2251,7 @@ tasks:
|
|||
- "./etc/repo_config.yaml"
|
||||
- "./etc/scons/**"
|
||||
- "buildscripts/**"
|
||||
- "version_expansions.yml"
|
||||
- "compile_expansions.yml"
|
||||
- "all_feature_flags.txt" # Must correspond to the definition in buildscripts/idl/lib.py.
|
||||
- "jstests/**"
|
||||
|
|
@ -2279,6 +2317,9 @@ tasks:
|
|||
|
||||
- name: compile_ninja
|
||||
tags: []
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- func: "scons compile"
|
||||
vars:
|
||||
|
|
@ -2297,6 +2338,9 @@ tasks:
|
|||
|
||||
- name: compile_ninja_next
|
||||
tags: []
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- func: "scons compile"
|
||||
vars:
|
||||
|
|
@ -2316,6 +2360,9 @@ tasks:
|
|||
|
||||
- name: compile_build_tools_next
|
||||
tags: []
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- func: "scons compile"
|
||||
vars:
|
||||
|
|
@ -2326,6 +2373,9 @@ tasks:
|
|||
|
||||
- name: libdeps_graph_linting
|
||||
tags: []
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- *f_expansions_write
|
||||
- command: subprocess.exec
|
||||
|
|
@ -2385,8 +2435,10 @@ tasks:
|
|||
- name: clang_tidy
|
||||
tags: []
|
||||
exec_timeout_secs: 3600 # 1 hour timeout for the task overall
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- *f_expansions_write
|
||||
- func: "scons compile"
|
||||
vars:
|
||||
targets: generated-sources compiledb
|
||||
|
|
@ -2433,7 +2485,7 @@ tasks:
|
|||
## run_unittests with UndoDB live-record ##
|
||||
#- name: run_unittests_with_recording
|
||||
# depends_on:
|
||||
# - name: compile_unittests_for_recorded_unittest
|
||||
# - name: compile_unittests_for_recorded_unittest
|
||||
# commands:
|
||||
# - *f_expansions_write
|
||||
# - func: "run diskstats"
|
||||
|
|
@ -2462,6 +2514,9 @@ tasks:
|
|||
##compile_and_archive_libfuzzertests - build libfuzzertests ##
|
||||
- name: compile_and_archive_libfuzzertests
|
||||
tags: []
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- func: "scons compile"
|
||||
vars:
|
||||
|
|
@ -2492,6 +2547,9 @@ tasks:
|
|||
|
||||
- name: server_discovery_and_monitoring_json_test
|
||||
tags: []
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- func: "scons compile"
|
||||
vars:
|
||||
|
|
@ -2504,6 +2562,9 @@ tasks:
|
|||
|
||||
- name: server_selection_json_test
|
||||
tags: []
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- func: "scons compile"
|
||||
vars:
|
||||
|
|
@ -2551,6 +2612,9 @@ tasks:
|
|||
|
||||
- name: compile_visibility_test
|
||||
tags: []
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- func: "scons compile"
|
||||
vars:
|
||||
|
|
@ -2578,7 +2642,12 @@ tasks:
|
|||
|
||||
- name: embedded_sdk_build_cdriver
|
||||
tags: []
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- *get_version_expansions
|
||||
- *apply_version_expansions
|
||||
- func: f_expansions_write
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
|
|
@ -2613,6 +2682,8 @@ tasks:
|
|||
depends_on:
|
||||
- name: embedded_sdk_install_dev
|
||||
commands:
|
||||
- *get_version_expansions
|
||||
- *apply_version_expansions
|
||||
- *f_expansions_write
|
||||
- command: subprocess.exec
|
||||
params:
|
||||
|
|
@ -2656,6 +2727,8 @@ tasks:
|
|||
depends_on:
|
||||
- name: embedded_sdk_install_tests
|
||||
commands:
|
||||
- *get_version_expansions
|
||||
- *apply_version_expansions
|
||||
- *f_expansions_write
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
|
|
@ -2682,6 +2755,8 @@ tasks:
|
|||
depends_on:
|
||||
- name: embedded_sdk_install_tests
|
||||
commands:
|
||||
- *get_version_expansions
|
||||
- *apply_version_expansions
|
||||
- *f_expansions_write
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
|
|
@ -2700,6 +2775,8 @@ tasks:
|
|||
depends_on:
|
||||
- name: embedded_sdk_run_tests
|
||||
commands:
|
||||
- *get_version_expansions
|
||||
- *apply_version_expansions
|
||||
# A second put, this time to -latest, to give devs a reasonable
|
||||
# way to get the most recent build.
|
||||
- command: s3.put
|
||||
|
|
@ -2719,6 +2796,8 @@ tasks:
|
|||
depends_on:
|
||||
- name: embedded_sdk_run_tests
|
||||
commands:
|
||||
- *get_version_expansions
|
||||
- *apply_version_expansions
|
||||
# A second put, this time to -latest, to give devs a reasonable
|
||||
# way to get the most recent build.
|
||||
- command: s3.put
|
||||
|
|
@ -2735,6 +2814,9 @@ tasks:
|
|||
|
||||
- name: stitch_support_create_lib
|
||||
tags: []
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- *f_expansions_write
|
||||
- func: "scons compile"
|
||||
|
|
@ -2785,6 +2867,8 @@ tasks:
|
|||
depends_on:
|
||||
- name: stitch_support_install_tests
|
||||
commands:
|
||||
- func: "get version expansions"
|
||||
- func: "apply version expansions"
|
||||
- *f_expansions_write
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
|
|
@ -2795,6 +2879,9 @@ tasks:
|
|||
|
||||
- name: crypt_create_lib
|
||||
tags: []
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- *f_expansions_write
|
||||
- func: "scons compile"
|
||||
|
|
@ -2828,6 +2915,9 @@ tasks:
|
|||
|
||||
- name: crypt_create_debug_lib
|
||||
tags: []
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- *f_expansions_write
|
||||
- func: "scons compile"
|
||||
|
|
@ -2898,6 +2988,8 @@ tasks:
|
|||
depends_on:
|
||||
- name: crypt_install_tests
|
||||
commands:
|
||||
- func: "get version expansions"
|
||||
- func: "apply version expansions"
|
||||
- *f_expansions_write
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
|
|
@ -2908,7 +3000,9 @@ tasks:
|
|||
|
||||
- name: compile_benchmarks
|
||||
tags: []
|
||||
depends_on: []
|
||||
depends_on:
|
||||
- name: version_expansions_gen
|
||||
variant: generate-tasks-for-version
|
||||
commands:
|
||||
- command: manifest.load
|
||||
- func: "git get project and add git tag"
|
||||
|
|
@ -4181,6 +4275,18 @@ tasks:
|
|||
- func: "upload pip requirements"
|
||||
- func: "generate version"
|
||||
|
||||
- name: version_expansions_gen
|
||||
commands:
|
||||
- command: manifest.load
|
||||
- *git_get_project
|
||||
- *f_expansions_write
|
||||
- *add_git_tag
|
||||
- *kill_processes
|
||||
- *cleanup_environment
|
||||
- func: "set up venv"
|
||||
- func: "upload pip requirements"
|
||||
- func: "generate version expansions"
|
||||
|
||||
- name: selected_tests_gen
|
||||
tags: []
|
||||
commands:
|
||||
|
|
@ -7553,6 +7659,7 @@ task_groups:
|
|||
tags: ["stitch"]
|
||||
tasks:
|
||||
- "stitch_support_create_lib"
|
||||
|
||||
- <<: *stitch_support_task_group_template
|
||||
name: stitch_support_lib_build_and_test
|
||||
tags: ["stitch"]
|
||||
|
|
|
|||
|
|
@ -9,3 +9,4 @@ buildvariants:
|
|||
- rhel80-medium
|
||||
tasks:
|
||||
- name: version_gen
|
||||
- name: version_expansions_gen
|
||||
|
|
|
|||
32
etc/perf.yml
32
etc/perf.yml
|
|
@ -244,11 +244,16 @@ functions:
|
|||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
mkdir -p mongodb/bin
|
||||
|
||||
/opt/mongodbtoolchain/v3/bin/virtualenv --python /opt/mongodbtoolchain/v3/bin/python3 "${workdir}/compile_venv"
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
|
||||
python -m pip install -r etc/pip/compile-requirements.txt
|
||||
|
||||
- command: expansions.write
|
||||
params:
|
||||
file: expansions.yml
|
||||
redacted: true
|
||||
- command: shell.exec
|
||||
params:
|
||||
working_dir: src
|
||||
|
|
@ -256,10 +261,10 @@ functions:
|
|||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
mkdir -p mongodb/bin
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
|
||||
# We get the raw version string (r1.2.3-45-gabcdef) from git
|
||||
MONGO_VERSION=$(git describe --abbrev=7)
|
||||
export MONGO_VERSION=$(git describe --abbrev=7)
|
||||
|
||||
# If this is a patch build, we add the patch version id to the version string so we know
|
||||
# this build was a patch, and which evergreen task it came from
|
||||
|
|
@ -267,10 +272,25 @@ functions:
|
|||
MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
|
||||
fi
|
||||
|
||||
# This script converts the generated version string into a sanitized version string for
|
||||
# use by scons and uploading artifacts as well as information about for the scons cache.
|
||||
# This script handles sanitizing the version string for use during SCons build
|
||||
# and when pushing artifacts up to S3.
|
||||
IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
|
||||
buildscripts/generate_version_expansions.py --out version_expansions.yml
|
||||
- command: expansions.update
|
||||
params:
|
||||
file: src/version_expansions.yml
|
||||
- command: shell.exec
|
||||
params:
|
||||
working_dir: src
|
||||
script: |
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
# This script handles whether the SCons cache should be used
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
MONGO_VERSION=$MONGO_VERSION USE_SCONS_CACHE=${use_scons_cache|false} python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
|
||||
SCONS_CACHE_MODE=${scons_cache_mode|} USE_SCONS_CACHE=${use_scons_cache|false} \
|
||||
IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
|
||||
python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
|
||||
- command: expansions.update
|
||||
params:
|
||||
file: src/compile_expansions.yml
|
||||
|
|
|
|||
|
|
@ -266,11 +266,16 @@ functions:
|
|||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
mkdir -p mongodb/bin
|
||||
|
||||
/opt/mongodbtoolchain/v3/bin/virtualenv --python /opt/mongodbtoolchain/v3/bin/python3 "${workdir}/compile_venv"
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
|
||||
python -m pip install -r etc/pip/compile-requirements.txt
|
||||
|
||||
- command: expansions.write
|
||||
params:
|
||||
file: expansions.yml
|
||||
redacted: true
|
||||
- command: shell.exec
|
||||
params:
|
||||
working_dir: src
|
||||
|
|
@ -278,10 +283,10 @@ functions:
|
|||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
mkdir -p mongodb/bin
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
|
||||
# We get the raw version string (r1.2.3-45-gabcdef) from git
|
||||
MONGO_VERSION=$(git describe --abbrev=7)
|
||||
export MONGO_VERSION=$(git describe --abbrev=7)
|
||||
|
||||
# If this is a patch build, we add the patch version id to the version string so we know
|
||||
# this build was a patch, and which evergreen task it came from
|
||||
|
|
@ -289,10 +294,25 @@ functions:
|
|||
MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
|
||||
fi
|
||||
|
||||
# This script converts the generated version string into a sanitized version string for
|
||||
# use by scons and uploading artifacts as well as information about for the scons cache.
|
||||
# This script handles sanitizing the version string for use during SCons build
|
||||
# and when pushing artifacts up to S3.
|
||||
IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
|
||||
buildscripts/generate_version_expansions.py --out version_expansions.yml
|
||||
- command: expansions.update
|
||||
params:
|
||||
file: src/version_expansions.yml
|
||||
- command: shell.exec
|
||||
params:
|
||||
working_dir: src
|
||||
script: |
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
# This script handles whether the SCons cache should be used
|
||||
source "${workdir}/compile_venv/bin/activate"
|
||||
MONGO_VERSION=$MONGO_VERSION USE_SCONS_CACHE=${use_scons_cache|false} python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
|
||||
SCONS_CACHE_MODE=${scons_cache_mode|} USE_SCONS_CACHE=${use_scons_cache|false} \
|
||||
IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
|
||||
python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
|
||||
- command: expansions.update
|
||||
params:
|
||||
file: src/compile_expansions.yml
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ cd src
|
|||
set -o errexit
|
||||
mongo_binary=dist-test/bin/mongo${exe}
|
||||
activate_venv
|
||||
bin_ver=$($python -c "import yaml; print(yaml.safe_load(open('compile_expansions.yml'))['version']);" | tr -d '[ \r\n]')
|
||||
bin_ver=$($python -c "import yaml; print(yaml.safe_load(open('version_expansions.yml'))['version']);" | tr -d '[ \r\n]')
|
||||
# Due to SERVER-23810, we cannot use $mongo_binary --quiet --nodb --eval "version();"
|
||||
mongo_ver=$($mongo_binary --version | perl -pe '/version v([^\"]*)/; $_ = $1;' | tr -d '[ \r\n]')
|
||||
# The versions must match
|
||||
|
|
|
|||
|
|
@ -5,14 +5,6 @@ cd src
|
|||
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
# We get the raw version string (r1.2.3-45-gabcdef) from git
|
||||
MONGO_VERSION=$(git describe --abbrev=7)
|
||||
# If this is a patch build, we add the patch version id to the version string so we know
|
||||
# this build was a patch, and which evergreen task it came from
|
||||
if [ "${is_patch}" = "true" ]; then
|
||||
MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
|
||||
fi
|
||||
echo "MONGO_VERSION = ${MONGO_VERSION}"
|
||||
activate_venv
|
||||
# shared scons cache testing
|
||||
# if 'scons_cache_scope' enabled and project level 'disable_shared_scons_cache' is not true
|
||||
|
|
@ -43,12 +35,12 @@ if [ ! -z ${scons_cache_scope} ]; then
|
|||
set -o errexit
|
||||
fi
|
||||
echo "Shared Cache with setting: ${scons_cache_scope}"
|
||||
MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode} SCONS_CACHE_SCOPE=$scons_cache_scope IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_compile_expansions_shared_cache.py --out compile_expansions.yml
|
||||
SCONS_CACHE_MODE=${scons_cache_mode} SCONS_CACHE_SCOPE=$scons_cache_scope IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} buildscripts/generate_compile_expansions_shared_cache.py --out compile_expansions.yml
|
||||
# Legacy Expansion generation
|
||||
else
|
||||
echo "Using legacy expansion generation"
|
||||
# Proceed with regular expansions generated
|
||||
# This script converts the generated version string into a sanitized version string for
|
||||
# use by scons and uploading artifacts as well as information about for the scons cache.
|
||||
MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode} USE_SCONS_CACHE=${use_scons_cache} $python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
|
||||
SCONS_CACHE_MODE=${scons_cache_mode} USE_SCONS_CACHE=${use_scons_cache} IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} buildscripts/generate_compile_expansions.py --out compile_expansions.yml
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
|
||||
. "$DIR/../prelude.sh"
|
||||
|
||||
cd src
|
||||
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
# We get the raw version string (r1.2.3-45-gabcdef) from git
|
||||
MONGO_VERSION=$(git describe --abbrev=7)
|
||||
# If this is a patch build, we add the patch version id to the version string so we know
|
||||
# this build was a patch, and which evergreen task it came from
|
||||
if [ "${is_patch}" = "true" ]; then
|
||||
MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
|
||||
fi
|
||||
echo "MONGO_VERSION = ${MONGO_VERSION}"
|
||||
|
||||
MONGO_VERSION=${MONGO_VERSION} IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} buildscripts/generate_version_expansions.py --out version_expansions.yml
|
||||
Loading…
Reference in New Issue