"""FCV and Server binary version constants used for multiversion testing.""" import glob import http import os import shutil from subprocess import DEVNULL, STDOUT, CalledProcessError, call, check_output import requests import structlog from retry import retry import buildscripts.resmokelib.config as _config from buildscripts.resmokelib.multiversion.multiversion_service import ( MongoReleases, MongoVersion, MultiversionService, ) from buildscripts.resmokelib.multiversionsetupconstants import USE_EXISTING_RELEASES_FILE from buildscripts.util.expansions import get_expansion LAST_LTS = "last_lts" LAST_CONTINUOUS = "last_continuous" RELEASES_LOCAL_FILE = os.path.join("src", "mongo", "util", "version", "releases.yml") # We use the "releases.yml" file from "master" because it is guaranteed to be up-to-date # with the latest EOL versions. If a "last-continuous" version is EOL, we don't include # it in the multiversion config and therefore don't test against it. MASTER_RELEASES_REMOTE_FILE = ( "https://raw.githubusercontent.com/mongodb/mongo/master/src/mongo/util/version/releases.yml" ) LOGGER = structlog.getLogger(__name__) def generate_mongo_version_file(): """Generate the mongo version data file. Should only be called in the root of the mongo directory.""" try: res = check_output("git describe", shell=True, text=True) except CalledProcessError as exp: raise ChildProcessError("Failed to run git describe to get the latest tag") from exp # Write the current MONGO_VERSION to a data file. with open(_config.MONGO_VERSION_FILE, "w") as mongo_version_fh: # E.g. res = 'r5.1.0-alpha-597-g8c345c6693\n' res = res[1:] # Remove the leading "r" character. mongo_version_fh.write("mongo_version: " + res) @retry(tries=5, delay=3) def get_releases_file_from_remote(): """Get the latest releases.yml from github.""" try: with open(_config.RELEASES_FILE, "wb") as file: response = requests.get(MASTER_RELEASES_REMOTE_FILE) if response.status_code != http.HTTPStatus.OK: raise RuntimeError( f"Fetching releases.yml file returned unsuccessful status: {response.status_code}, " f"response body: {response.text}\n" ) file.write(response.content) LOGGER.info(f"Got releases.yml file remotely: {MASTER_RELEASES_REMOTE_FILE}") except Exception as exc: LOGGER.warning(f"Could not get releases.yml file remotely: {MASTER_RELEASES_REMOTE_FILE}") raise exc def get_releases_file_locally_or_fallback_to_remote(): """Get the latest releases.yml locally or fallback to getting it from github.""" if os.path.exists(RELEASES_LOCAL_FILE): LOGGER.info(f"Found releases.yml file locally: {RELEASES_LOCAL_FILE}") shutil.copyfile(RELEASES_LOCAL_FILE, _config.RELEASES_FILE) else: LOGGER.warning(f"Could not find releases.yml file locally: {RELEASES_LOCAL_FILE}") get_releases_file_from_remote() def generate_releases_file(): """Generate the releases constants file.""" # If we are not in master, we want to grab the releases file from master as a source of truth. # If we are on master in CI, we can grab it locally to accommodate any changes made for testing. # On older versions we use the master releases.yml file to ensure we do not run multiversion # testing against eol versions. if _config.EVERGREEN_TASK_ID and get_expansion("branch_name") != "master": get_releases_file_from_remote() else: get_releases_file_locally_or_fallback_to_remote() def in_git_root_dir(): """Return True if we are in the root of a git directory.""" try: if call(["git", "branch"], stderr=STDOUT, stdout=DEVNULL) != 0: # We are not in a git directory. return False except FileNotFoundError: # Git is not even installed. return False git_root_dir = os.path.realpath( check_output("git rev-parse --show-toplevel", shell=True, text=True).strip() ) curr_dir = os.path.realpath(os.getcwd()) return git_root_dir == curr_dir # Check for the source dir that exists on spawnhosts def in_spawn_host(): if glob.glob("/data/mci/source*"): return True return False if (in_git_root_dir() or "BUILD_WORKSPACE_DIRECTORY" in os.environ) and not in_spawn_host(): generate_mongo_version_file() else: LOGGER.info("Skipping generating mongo version file since we're not in the root of a git repo") # Avoiding regenerating the releases file if this flag is set. Should only be set if there are # multiple processes attempting to set up multiversion concurrently. if not USE_EXISTING_RELEASES_FILE: generate_releases_file() else: LOGGER.info( "Skipping generating releases file since the --useExistingReleasesFile flag has been set" ) def evg_project_str(version): """Return the evergreen project name for the given version.""" return "mongodb-mongo-v{}.{}".format(version.major, version.minor) multiversion_service = MultiversionService( mongo_version=MongoVersion.from_yaml_file(_config.MONGO_VERSION_FILE), mongo_releases=MongoReleases.from_yaml_file(_config.RELEASES_FILE), ) version_constants = multiversion_service.calculate_version_constants() LAST_LTS_BIN_VERSION = version_constants.get_last_lts_fcv() LAST_CONTINUOUS_BIN_VERSION = version_constants.get_last_continuous_fcv() LAST_LTS_FCV = version_constants.get_last_lts_fcv() LAST_CONTINUOUS_FCV = version_constants.get_last_continuous_fcv() LATEST_FCV = version_constants.get_latest_fcv() LAST_CONTINUOUS_MONGO_BINARY = version_constants.build_last_continuous_binary("mongo") LAST_CONTINUOUS_MONGOD_BINARY = version_constants.build_last_continuous_binary("mongod") LAST_CONTINUOUS_MONGOS_BINARY = version_constants.build_last_continuous_binary("mongos") LAST_LTS_MONGO_BINARY = version_constants.build_last_lts_binary("mongo") LAST_LTS_MONGOD_BINARY = version_constants.build_last_lts_binary("mongod") LAST_LTS_MONGOS_BINARY = version_constants.build_last_lts_binary("mongos") REQUIRES_FCV_TAG_LATEST = version_constants.get_latest_tag() # Generate tags for all FCVS in (lastLTS, latest]. # All multiversion tests should be run with these tags excluded. REQUIRES_FCV_TAG = version_constants.get_fcv_tag_list() REQUIRES_FCV_TAGS_LESS_THAN_LATEST = version_constants.get_fcv_tags_less_than_latest() # Generate evergreen project names for all FCVs less than latest. EVERGREEN_PROJECTS = ["mongodb-mongo-master"] EVERGREEN_PROJECTS.extend([evg_project_str(fcv) for fcv in version_constants.fcvs_less_than_latest]) OLD_VERSIONS = ( [LAST_LTS] if LAST_CONTINUOUS_FCV == LAST_LTS_FCV or LAST_CONTINUOUS_FCV in version_constants.get_eols() else [LAST_LTS, LAST_CONTINUOUS] ) def log_constants(exec_log): """Log FCV constants.""" exec_log.info("Last LTS FCV: {}".format(LAST_LTS_FCV)) exec_log.info("Last Continuous FCV: {}".format(LAST_CONTINUOUS_FCV)) exec_log.info("Latest FCV: {}".format(LATEST_FCV)) exec_log.info("Requires FCV Tag Latest: {}".format(REQUIRES_FCV_TAG_LATEST)) exec_log.info("Requires FCV Tag: {}".format(REQUIRES_FCV_TAG))