import os import shlex import shutil import subprocess import sys import git import yaml from buildscripts.resmokelib import config from buildscripts.resmokelib.errors import RequiresForceRemove from buildscripts.util.expansions import get_expansion def build_images(suite_name, fixture_instance): """Build images needed to run the resmoke suite against docker containers.""" image_builder = DockerComposeImageBuilder(suite_name, fixture_instance) if "config" in config.DOCKER_COMPOSE_BUILD_IMAGES: image_builder.build_config_image() if "mongo-binaries" in config.DOCKER_COMPOSE_BUILD_IMAGES: image_builder.build_mongo_binaries_image() if "workload" in config.DOCKER_COMPOSE_BUILD_IMAGES: image_builder.build_workload_image() if config.DOCKER_COMPOSE_BUILD_IMAGES: repro_command = f""" Built image(s): {config.DOCKER_COMPOSE_BUILD_IMAGES} SUCCESS - Run this suite against an External System Under Test (SUT) with the following command: `docker compose -f docker_compose/{suite_name}/docker-compose.yml run --rm workload {image_builder.get_resmoke_run_command()}` DISCLAIMER - Make sure you have built all images with the following command first: `buildscripts/resmoke.py run --suite {suite_name} --dockerComposeBuildImages workload,mongo-binaries,config` """ print(repro_command) class DockerComposeImageBuilder: """Build images needed to run a resmoke suite against a MongoDB Docker Container topology.""" def __init__(self, suite_name, suite_fixture): """ Constructs a `DockerComposeImageBuilder` which can build images locally and in CI. :param suite_name: The name of the suite we are building images for. :param suite_fixture: The fixture to base the `docker-compose.yml` generation off of. """ self.suite_name = suite_name self.suite_fixture = suite_fixture self.tag = config.DOCKER_COMPOSE_TAG self.in_evergreen = config.DOCKER_COMPOSE_BUILD_ENV == "evergreen" # Build context constants self.DOCKER_COMPOSE_BUILD_CONTEXT = f"docker_compose/{self.suite_name}" self.WORKLOAD_BUILD_CONTEXT = "buildscripts/antithesis/base_images/workload" self.WORKLOAD_DOCKERFILE = f"{self.WORKLOAD_BUILD_CONTEXT}/Dockerfile" self.MONGO_BINARIES_BUILD_CONTEXT = "buildscripts/antithesis/base_images/mongo_binaries" self.MONGO_BINARIES_DOCKERFILE = f"{self.MONGO_BINARIES_BUILD_CONTEXT}/Dockerfile" # Artifact constants self.DIST_TEST_DIR = "dist-test" if self.in_evergreen else "antithesis-dist-test" self.MONGODB_BINARIES_DIR = os.path.join(self.DIST_TEST_DIR, "bin") self.MONGODB_LIBRARIES_DIR = os.path.join(self.DIST_TEST_DIR, "lib") self.TSAN_SUPPRESSIONS_SOURCE = "etc/tsan.suppressions" self.MONGO_BINARY = os.path.join(self.MONGODB_BINARIES_DIR, "mongo") self.MONGOD_BINARY = os.path.join(self.MONGODB_BINARIES_DIR, "mongod") self.MONGOS_BINARY = os.path.join(self.MONGODB_BINARIES_DIR, "mongos") self.LIBVOIDSTAR_PATH = "/usr/lib/libvoidstar.so" self.MONGODB_DEBUGSYMBOLS = "mongo-debugsymbols.tgz" # MongoDB Enterprise Modules constants self.MODULES_RELATIVE_PATH = "src/mongo/db/modules" self.MONGO_ENTERPRISE_MODULES_RELATIVE_PATH = f"{self.MODULES_RELATIVE_PATH}/enterprise" # Port suffix ranging from 1-24 is subject to fault injection while ports 130+ are safe. self.next_available_fault_enabled_ip = 2 self.next_available_fault_disabled_ip = 130 self.san_options = self._get_san_options() def _get_san_options(self): if self.in_evergreen: san_options = get_expansion("san_options", "") else: san_options = os.environ.get("san_options", "") lines = [line for line in san_options.split() if line] options = {} for line in lines: lst = [x.strip() for x in line.split("=", maxsplit=1)] assert len(lst) == 2, f"Failed to parse $san_options: {line}" k, v = lst if v.startswith(('"', "'")) or v.endswith(("'", '"')): assert v[0] == v[-1], f"Failed to parse $san_options: {line}" v = v[1:-1] options[k] = v return options @staticmethod def get_resmoke_run_command() -> str: """Construct the supported resmoke `run` command to test against an external system under test.""" # The supported `run` command should keep all of the same args except: # (1) it should remove the `--dockerComposeBuildImages` option and value # (2) it should add the `--externalSUT` flag command = sys.argv rm_index = command.index("--dockerComposeBuildImages") return " ".join(command[0:rm_index] + command[rm_index + 2 :] + ["--externalSUT"]) def _add_docker_compose_configuration_to_build_context(self, build_context) -> None: """ Create init scripts for all of the mongo{d,s} processes and a `docker-compose.yml` file. :param build_context: Filepath where the configuration is going to be set up. """ def create_docker_compose_service(name, fault_injection, depends_on): """ Create a service section of a docker-compose.yml for a service with this name. :param name: Whether or not this service should be subject to fault injection. :param fault_injection: Whether or not this service should be subject to fault injection. :param depends_on: Any services that this service depends on to successfully run. """ if fault_injection: ip_suffix = self.next_available_fault_enabled_ip self.next_available_fault_enabled_ip += 1 else: ip_suffix = self.next_available_fault_disabled_ip self.next_available_fault_disabled_ip += 1 return { "container_name": name, "hostname": name, "image": f'{"workload" if name == "workload" else "mongo-binaries"}:{self.tag}', "volumes": [ f"./logs/{name}:/var/log/mongodb/", "./scripts:/scripts/", f"./data/{name}:/data/db", ], "command": f"/bin/bash /scripts/{name}.sh", "networks": {"antithesis-net": {"ipv4_address": f"10.20.20.{ip_suffix}"}}, "depends_on": depends_on, } docker_compose_yml = { "version": "3.0", "services": { "workload": create_docker_compose_service( "workload", fault_injection=False, depends_on=[ process.logger.external_sut_hostname for process in self.suite_fixture.all_processes() ], ) }, "networks": { "antithesis-net": { "driver": "bridge", "ipam": {"config": [{"subnet": "10.20.20.0/24"}]}, } }, } print("Writing workload init script...") with open(os.path.join(build_context, "scripts", "workload.sh"), "w") as workload_init: workload_init.write("tail -f /dev/null\n") print("Writing resmoke run script for convenience...") with open(os.path.join(build_context, "scripts", "run_resmoke.sh"), "w") as run_resmoke: run_resmoke.write(f'{self.get_resmoke_run_command()} "$@"\n') print("Writing mongo{d,s} init scripts...") for process in self.suite_fixture.all_processes(): # Add the `Process` as a service in the docker-compose.yml service_name = process.logger.external_sut_hostname docker_compose_yml["services"][service_name] = create_docker_compose_service( service_name, fault_injection=True, depends_on=[] ) # Write the `Process` args as an init script with open(os.path.join(build_context, "scripts", f"{service_name}.sh"), "w") as file: file.write(" ".join(map(shlex.quote, process.args)) + "\n") print("Writing `docker-compose.yml`...") with open(os.path.join(build_context, "docker-compose.yml"), "w") as docker_compose: docker_compose.write(yaml.dump(docker_compose_yml) + "\n") print("Writing Dockerfile...") with open(os.path.join(build_context, "Dockerfile"), "w") as dockerfile: dockerfile.write("FROM scratch\n") dockerfile.write("COPY docker-compose.yml /\n") dockerfile.write("ADD scripts /scripts\n") dockerfile.write("ADD logs /logs\n") dockerfile.write("ADD data /data\n") dockerfile.write("ADD debug /debug\n") def _initialize_docker_compose_build_context(self, build_context) -> None: """ Remove the old docker compose build context and create a new one. :param build_context: Filepath where the configuration is going to be set up. """ try: shutil.rmtree(build_context) except FileNotFoundError as _: # `shutil.rmtree` throws FileNotFoundError if the path DNE. In that case continue as normal. pass except Exception as exc: exception_text = f""" Could not remove directory due to old artifacts from a previous run. Please remove this directory and try again -- you may need to force remove: `{os.path.relpath(build_context)}` """ raise RequiresForceRemove(exception_text) from exc for volume in ["scripts", "logs", "data", "debug"]: os.makedirs(os.path.join(build_context, volume)) def _get_docker_build_san_args(self): args = [] for key, value in self.san_options.items(): args += ["--build-arg", f"{key}={value}"] return args def _docker_build(self, image_name, dockerfile, build_context): cmd = ["docker", "build", "-t", f"{image_name}:{self.tag}", "-f", dockerfile] cmd += self._get_docker_build_san_args() cmd += [build_context] print(f"Building image: {shlex.join(cmd)}") return subprocess.run(cmd, stdout=sys.stdout, stderr=sys.stderr, check=True) def build_config_image(self): """ Build the config image containing the `docker-compose.yml` file, init scripts and volumes for the suite. :return: None """ # Build out the directory structure and write the startup scripts for the config image print(f"Preparing antithesis config image build context for `{self.suite_name}`...") self._initialize_docker_compose_build_context(self.DOCKER_COMPOSE_BUILD_CONTEXT) self._add_docker_compose_configuration_to_build_context(self.DOCKER_COMPOSE_BUILD_CONTEXT) # Our official builds happen in Evergreen. Assert debug symbols are on system. # If this is running locally, this is for development purposes only and debug symbols are not required. if self.in_evergreen: assert os.path.exists( self.MONGODB_DEBUGSYMBOLS ), f"No debug symbols available at: {self.MONGODB_DEBUGSYMBOLS}" print("Running in Evergreen -- copying debug symbols to build context...") shutil.copy( self.MONGODB_DEBUGSYMBOLS, os.path.join(self.DOCKER_COMPOSE_BUILD_CONTEXT, "debug") ) print(f"Done setting up antithesis config image build context for `{self.suite_name}...") print("Building antithesis config image...") self._docker_build( self.suite_name, f"{self.DOCKER_COMPOSE_BUILD_CONTEXT}/Dockerfile", self.DOCKER_COMPOSE_BUILD_CONTEXT, ) print("Done building antithesis config image.") def build_workload_image(self): """ Build the workload image. :param tag: Tag to use for the image. :return: None. """ assert os.path.exists( self.MONGO_ENTERPRISE_MODULES_RELATIVE_PATH ), f"Please set up `mongo_enterprise_modules` and try again. No `mongo_enterprise_modules` repo available at: {self.MONGO_ENTERPRISE_MODULES_RELATIVE_PATH}" print("Prepping `workload` image build context...") # Set up build context self._fetch_mongodb_binaries() self._copy_mongodb_libraries_to_build_context(self.WORKLOAD_BUILD_CONTEXT) self._copy_mongo_binary_to_build_context(self.WORKLOAD_BUILD_CONTEXT) self._clone_mongo_repo_to_build_context(self.WORKLOAD_BUILD_CONTEXT) self._clone_qa_repo_to_build_context(self.WORKLOAD_BUILD_CONTEXT) self._clone_jstestfuzz_to_build_context(self.WORKLOAD_BUILD_CONTEXT) self._add_libvoidstar_to_build_context(self.WORKLOAD_BUILD_CONTEXT) self._copy_config_files_to_build_context(self.WORKLOAD_BUILD_CONTEXT) # Build docker image print("Building workload image...") self._docker_build("workload", self.WORKLOAD_DOCKERFILE, self.WORKLOAD_BUILD_CONTEXT) print("Done building workload image.") def build_mongo_binaries_image(self): """ Build the mongo-binaries image. :return: None. """ # Set up build context print("Prepping `mongo binaries` image build context...") self._fetch_mongodb_binaries() self._copy_mongodb_libraries_to_build_context(self.MONGO_BINARIES_BUILD_CONTEXT) self._copy_mongodb_binaries_to_build_context(self.MONGO_BINARIES_BUILD_CONTEXT) self._clone_mongo_repo_to_build_context(self.MONGO_BINARIES_BUILD_CONTEXT) self._add_libvoidstar_to_build_context(self.MONGO_BINARIES_BUILD_CONTEXT) self._copy_config_files_to_build_context(self.MONGO_BINARIES_BUILD_CONTEXT) # Build docker image print("Building mongo binaries image...") self._docker_build( "mongo-binaries", self.MONGO_BINARIES_DOCKERFILE, self.MONGO_BINARIES_BUILD_CONTEXT ) print("Done building mongo binaries image.") def _fetch_mongodb_binaries(self): """ Get MongoDB binaries and verify existence/validity. In CI the binaries from the `master` branch should already exist from the compile task. In that case, we just want to fetch the `last-continuous` and `last-lts` binaries. :return: None. """ mongodb_binaries_destination = self.MONGODB_BINARIES_DIR if not self.in_evergreen and os.path.exists(mongodb_binaries_destination): print( f"\n\tRunning Locally - Found existing MongoDB binaries at: {mongodb_binaries_destination}\n" ) # If local, fetch the binaries. elif not self.in_evergreen: # Ensure that `db-contrib-tool` is installed locally db_contrib_tool_error = """ Could not find `db-contrib-tool` installation locally. You can install `db-contrib-tool` with the following command: `pip install db-contrib-tool` OR `pipx install db-contrib-tool` More info on `db-contrib-tool` available here: `https://github.com/10gen/db-contrib-tool` """ assert ( subprocess.run(["which", "db-contrib-tool"]).returncode == 0 ), db_contrib_tool_error # Use `db-contrib-tool` to get MongoDB binaries for this image print("Running Locally - Fetching All MongoDB binaries for image build...") subprocess.run( [ "db-contrib-tool", "setup-repro-env", "--variant", "ubuntu2204", "--linkDir", mongodb_binaries_destination, "--installLastContinuous", "--installLastLTS", "master", ], stdout=sys.stdout, stderr=sys.stderr, check=True, ) elif self.in_evergreen: print( "Running in Evergreen - Fetching `last-continuous` and `last-lts` MongoDB binaries for image build..." ) subprocess.run( [ "db-contrib-tool", "setup-repro-env", "--variant", "ubuntu2204", "--linkDir", mongodb_binaries_destination, "--installLastContinuous", "--installLastLTS", "--evergreenConfig", "./.evergreen.yml", ], stdout=sys.stdout, stderr=sys.stderr, check=True, ) # Verify the binaries were downloaded successfully for required_binary in [self.MONGO_BINARY, self.MONGOD_BINARY, self.MONGOS_BINARY]: assert os.path.exists( required_binary ), f"Could not find Ubuntu 22.04 MongoDB binary at: {required_binary}" # Our official builds happen in Evergreen. # We want to ensure the binaries are linked with `libvoidstar.so` during image build. if self.in_evergreen: assert "libvoidstar" in subprocess.run( ["ldd", required_binary], check=True, capture_output=True, ).stdout.decode( "utf-8" ), f"MongoDB binary is not linked to `libvoidstar.so`: {required_binary}" def _copy_mongo_binary_to_build_context(self, dir_path): """ Copy the mongo binary to the build context. :param dir_path: Directory path to add mongo binary to. """ mongo_binary_destination_dir = os.path.join(dir_path, "bin") mongo_binary_destination = os.path.join(mongo_binary_destination_dir, "mongo") print("Copy mongo binary to build context...") # Clean up any old artifacts in the build context if os.path.exists(mongo_binary_destination): print("Cleaning up mongo binary from build context first...") os.remove(mongo_binary_destination) os.makedirs(mongo_binary_destination_dir, exist_ok=True) shutil.copy(self.MONGO_BINARY, mongo_binary_destination) print("Done copying mongo binary to build context.") def _clone_mongo_repo_to_build_context(self, dir_path): """ Clone the mongo repo to the build context. :param dir_path: Directory path to clone mongo repo to. """ mongo_repo_destination = os.path.join(dir_path, "src") # Our official builds happen in Evergreen. Assert there's already a `mongo` repo in the build context. # This is because we cannot rely on the standard "git clone" command to include uncommitted changes applied from a `patch`. # Instead, we rely on Evergreen's `git.get_project` which will correctly clone the repo and apply changes from the `patch`. if self.in_evergreen: assert os.path.exists( mongo_repo_destination ), f"No `mongo` repo available at: {mongo_repo_destination}" print("Running in Evergreen -- no need to clone `mongo` repo since it already exists.") return # Clean up any old artifacts in the build context. if os.path.exists(mongo_repo_destination): print("Cleaning up MongoDB repo from build context first...") shutil.rmtree(mongo_repo_destination) print("Cloning current MongoDB repo to build context...") clone_repo_warning_message = """ - If you have uncommitted changes, they will not be included in the `workload` image. - If you would like to include these changes, commit your changes and try again. """ print(clone_repo_warning_message) # Copy the mongo repo to the build context. # If this fails to clone, the `git` library will raise an exception. active_branch = git.Repo("./").active_branch.name git.Repo.clone_from("./", mongo_repo_destination, branch=active_branch) print("Done cloning MongoDB repo to build context.") def _clone_qa_repo_to_build_context(self, dir_path): """ Clone the QA repo to the build context. :param dir_path: Directory path to clone QA repo to. """ qa_repo_destination = os.path.join(dir_path, "QA") # Clone QA repo if it does not already exist if os.path.exists(qa_repo_destination): print(f"\n\tFound existing QA repo at: {qa_repo_destination}\n") else: print("Cloning QA repo to build context...") self._clone_repo("10gen", "QA", qa_repo_destination, get_expansion("github_token_qa")) print("Done cloning QA repo to build context.") def _clone_jstestfuzz_to_build_context(self, dir_path): """ Clone the jstestfuzz repo to the build context. :param dir_path: Directory path to clone jstestfuzz repo to. """ jstestfuzz_repo_destination = os.path.join(dir_path, "jstestfuzz") # Clone jstestfuzz repo if it does not already exist if os.path.exists(jstestfuzz_repo_destination): print(f"\n\tFound existing jstestfuzz repo at: {jstestfuzz_repo_destination}\n") else: print("Cloning jstestfuzz repo to build context...") self._clone_repo( "10gen", "jstestfuzz", jstestfuzz_repo_destination, get_expansion("github_token_jstestfuzz"), ) print("Done cloning jstestfuzz repo to build context.") def _copy_config_files_to_build_context(self, dir_path): """ Copy any additional config files needed into the build context. :param dir_path: Directory path to copy config files to. """ shutil.copy(self.TSAN_SUPPRESSIONS_SOURCE, dir_path) def _copy_mongodb_binaries_to_build_context(self, dir_path): """ Copy the MongodDB binaries to the build context. :param dir_path: Directory path to add MongoDB binaries to. """ print("Copy all MongoDB binaries to build context...") mongodb_binaries_destination = os.path.join(dir_path, "bin") # Clean up any old artifacts in the build context if os.path.exists(mongodb_binaries_destination): print("Cleaning up all MongoDB binaries from build context first...") shutil.rmtree(mongodb_binaries_destination) shutil.copytree(self.MONGODB_BINARIES_DIR, mongodb_binaries_destination) print("Done copying all MongoDB binaries to build context.") def _copy_mongodb_libraries_to_build_context(self, dir_path): print("Copy all MongoDB libraries to build context...") mongodb_libraries_destination = os.path.join(dir_path, "lib") if os.path.exists(mongodb_libraries_destination): print("Cleaning up all MongoDB libaries from build context first...") shutil.rmtree(mongodb_libraries_destination) if os.path.exists(self.MONGODB_LIBRARIES_DIR): shutil.copytree(self.MONGODB_LIBRARIES_DIR, mongodb_libraries_destination) # We create lib/empty so that the relevant COPY in the Dockerfiles always has # something to copy - the docker build step will fail if it doesn't. This is important # when we're using a statically-linked build. mongodb_libs_dir = os.path.join(mongodb_libraries_destination, "empty") os.makedirs(mongodb_libs_dir, exist_ok=True) print("Done copying all MongoDB libraries to build context.") def _add_libvoidstar_to_build_context(self, dir_path): """ Add the antithesis instrumentation library from the system to the build context. :param dir_path: Directory path to add `libvoidstar.so` to. """ print("Add `libvoidstar.so` to build context...") libvoidstar_destination = os.path.join(dir_path, "libvoidstar.so") # Clean up any old artifacts in the build context if os.path.exists(libvoidstar_destination): print("Cleaning up `libvoidstar.so` from build context first...") os.remove(libvoidstar_destination) # Our official builds happen in Evergreen. Assert a "real" `libvoidstar.so` is on system. if self.in_evergreen: assert os.path.exists( self.LIBVOIDSTAR_PATH ), f"No `libvoidstar.so` available at: {self.LIBVOIDSTAR_PATH}" print("Running in Evergreen -- using system `libvoidstar.so`.") shutil.copy(self.LIBVOIDSTAR_PATH, libvoidstar_destination) print("Done copying `libvoidstar.so` from system to build context") else: disclaimer_message = """ This is a stub `libvoidstar.so` file. It does not actually do anything. For local development, we don't expect developers to have `libvoidstar.so` available, but it is necessary for building the base images and is part of the base image Dockerfile(s). """ with open(libvoidstar_destination, "w") as file: file.write(disclaimer_message) print( "Done writing stub `libvoidstar.so` to build context -- This is for development only." ) def _clone_repo(self, owner, repo, destination, token): """ Conditionally clone using either https or ssh depending on if running in evergreen. """ if token: print(f"Found token for {owner}/{repo} git repo, using http clone") url = f"https://x-access-token:{token}@github.com/{owner}/{repo}.git" else: print(f"No token found for {owner}/{repo} git repo, using ssh clone") assert not self.in_evergreen, "SSH cloning should only be done when not in evergreen" url = f"git@github.com:{owner}/{repo}.git" git.Repo.clone_from(url, destination)