From b4ceac3b6bbda8d0456118caeea29aef58109e69 Mon Sep 17 00:00:00 2001 From: Zack Winter <3457246+zackwintermdb@users.noreply.github.com> Date: Thu, 25 Sep 2025 11:19:37 -0700 Subject: [PATCH] SERVER-111404 Add rbe sysroot generation to spawnhost script (#41793) GitOrigin-RevId: f0aacf12838c0d9e741b75e68b888da5007cb7c9 --- bazel/platforms/BUILD.bazel | 4 + buildscripts/BUILD.bazel | 26 ++++++ buildscripts/create_rbe_sysroot.py | 43 +++++++++ buildscripts/local_rbe_container_url.py | 89 +++++++++++++++++++ buildscripts/print_local_rbe_container_url.py | 18 ++++ buildscripts/setup_spawnhost_coredump | 8 ++ 6 files changed, 188 insertions(+) create mode 100644 buildscripts/create_rbe_sysroot.py create mode 100644 buildscripts/local_rbe_container_url.py create mode 100644 buildscripts/print_local_rbe_container_url.py diff --git a/bazel/platforms/BUILD.bazel b/bazel/platforms/BUILD.bazel index 191a530eab5..e15dffce677 100644 --- a/bazel/platforms/BUILD.bazel +++ b/bazel/platforms/BUILD.bazel @@ -152,3 +152,7 @@ py_binary( srcs = ["remote_execution_containers_generator.py"], visibility = ["//visibility:public"], ) + +exports_files([ + "remote_execution_containers.bzl", +]) diff --git a/buildscripts/BUILD.bazel b/buildscripts/BUILD.bazel index 53ca7953ba8..0d70afb668b 100644 --- a/buildscripts/BUILD.bazel +++ b/buildscripts/BUILD.bazel @@ -373,3 +373,29 @@ py_binary( ), ], ) + +py_library( + name = "local_rbe_container_url", + srcs = ["local_rbe_container_url.py"], + data = [ + "//bazel/platforms:remote_execution_containers.bzl", + ], +) + +py_binary( + name = "print_local_rbe_container_url", + srcs = ["print_local_rbe_container_url.py"], + deps = [ + "local_rbe_container_url", + ], +) + +py_binary( + name = "create_rbe_sysroot", + srcs = [ + "create_rbe_sysroot.py", + ], + deps = [ + "local_rbe_container_url", + ], +) diff --git a/buildscripts/create_rbe_sysroot.py b/buildscripts/create_rbe_sysroot.py new file mode 100644 index 00000000000..9ea281c6774 --- /dev/null +++ b/buildscripts/create_rbe_sysroot.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import getpass +import os +import shutil +import subprocess +import sys + +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from buildscripts.local_rbe_container_url import calculate_local_rbe_container_url + + +def main(): + os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", ".")) + container_url = calculate_local_rbe_container_url() + if container_url == "UNKNOWN": + print("Could not determine local RBE container URL, cannot create rbe sysroot") + return 1 + + print(f"Using local RBE container URL: {container_url}") + + container_cli = shutil.which("docker") or shutil.which("podman") + if not container_cli: + print("Error: Neither docker nor podman is installed.", file=sys.stderr) + sys.exit(1) + + cid = subprocess.check_output([container_cli, "create", container_url]).decode().strip() + + os.makedirs("./rbe_sysroot", exist_ok=True) + + subprocess.run(["sudo", container_cli, "cp", f"{cid}:/", "./rbe_sysroot/"], check=True) + + user = getpass.getuser() + subprocess.run(["sudo", "chown", "-R", f"{user}:{user}", "./rbe_sysroot"], check=True) + subprocess.run([container_cli, "rm", cid], check=True) + + return 0 + + +if __name__ == "__main__": + exit(main()) diff --git a/buildscripts/local_rbe_container_url.py b/buildscripts/local_rbe_container_url.py new file mode 100644 index 00000000000..9db8e1b981d --- /dev/null +++ b/buildscripts/local_rbe_container_url.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 + +import os +import pathlib +import subprocess +import sys + +# Pulled from bazel/utils.bzl +_DISTRO_PATTERN_MAP = { + "Ubuntu 18*": "ubuntu18", + "Ubuntu 20*": "ubuntu20", + "Ubuntu 22*": "ubuntu22", + "Pop!_OS 22*": "ubuntu22", + "Ubuntu 24*": "ubuntu24", + "Amazon Linux 2": "amazon_linux_2", + "Amazon Linux 2023": "amazon_linux_2023", + "Debian GNU/Linux 10": "debian10", + "Debian GNU/Linux 12": "debian12", + "Red Hat Enterprise Linux 8*": "rhel8", + "Red Hat Enterprise Linux 9*": "rhel9", + "SLES 15*": "suse15", +} + + +def get_host_distro_major_version(): + # This code in this function looks a little janky in python. + # That's because it was pulled from starlark in bazel/utils.bzl to make sure the logic matches exactly. + if sys.platform != "linux": + return "UNKNOWN" + + result = subprocess.run( + [ + "sed", + "-n", + '/^\\(NAME\\|VERSION_ID\\)=/{s/[^=]*=//;s/"//g;p}', + "/etc/os-release", + ], + check=True, + capture_output=True, + text=True, + ) + + if result.returncode != 0: + print( + f"Failed to determine system distro, parsing os-release failed with the error: {result.stderr}" + ) + return "UNKNOWN" + + distro_seq = result.stdout.splitlines() + if len(distro_seq) != 2: + print(f"Failed to determine system distro, parsing os-release returned: {result.stdout}") + return "UNKNOWN" + + distro_str = "{distro_name} {distro_version}".format( + distro_name=distro_seq[0], + distro_version=distro_seq[1], + ) + + for distro_pattern, simplified_name in _DISTRO_PATTERN_MAP.items(): + if "*" in distro_pattern: + prefix_suffix = distro_pattern.split("*") + if distro_str.startswith(prefix_suffix[0]) and distro_str.endswith(prefix_suffix[1]): + return simplified_name + elif distro_str == distro_pattern: + return simplified_name + return "UNKNOWN" + + +def calculate_local_rbe_container_url(): + remote_execution_containers = {} + container_file_path = os.path.join( + pathlib.Path(__file__).parent.parent.resolve(), + "bazel", + "platforms", + "remote_execution_containers.bzl", + ) + with open(container_file_path, "r", encoding="utf-8") as f: + code = compile(f.read(), container_file_path, "exec") + exec(code, {}, remote_execution_containers) + host_distro = get_host_distro_major_version() + if host_distro == "UNKNOWN": + print("Could not determine host distro, cannot determine local RBE container URL") + return "UNKNOWN" + if host_distro not in remote_execution_containers["REMOTE_EXECUTION_CONTAINERS"]: + print(f"Host distro '{host_distro}' does not have a corresponding RBE container") + return "UNKNOWN" + return remote_execution_containers["REMOTE_EXECUTION_CONTAINERS"][host_distro][ + "container-url" + ].replace("docker://", "") diff --git a/buildscripts/print_local_rbe_container_url.py b/buildscripts/print_local_rbe_container_url.py new file mode 100644 index 00000000000..7acee88546a --- /dev/null +++ b/buildscripts/print_local_rbe_container_url.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import os +import sys + +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from buildscripts.print_local_rbe_container_url import calculate_local_rbe_container_url + + +def main(): + os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", ".")) + print(calculate_local_rbe_container_url()) + + +if __name__ == "__main__": + exit(main()) diff --git a/buildscripts/setup_spawnhost_coredump b/buildscripts/setup_spawnhost_coredump index c2bf450db62..10adbb20930 100755 --- a/buildscripts/setup_spawnhost_coredump +++ b/buildscripts/setup_spawnhost_coredump @@ -259,6 +259,13 @@ fi' >> .bash_profile archive_fail "src" fi + RBE_SYSROOT_SCRIPT="${SRC_DIR}/buildscripts/create_rbe_sysroot.py" + if [[ -f "${RBE_SYSROOT_SCRIPT}" ]]; then + echo "Setting up RBE sysroot for unit test debugging..." + $python $RBE_SYSROOT_SCRIPT + else + echo "Failed to find RBE sysroot setup script." + fi set -x @@ -398,4 +405,5 @@ fi # to error. if [[ -n "${slack_user}" ]]; then "$evg_binary_pathname" --config "$evg_credentials_pathname" notify slack -t "@$slack_user" -m "$slack_message" + "$evg_binary_pathname" --config "$evg_credentials_pathname" notify slack -t "@$slack_user" -m "If you run into unresolved system library symbols when debugging unit tests, please add -ex 'set sysroot $HOME_DIR/rbe_sysroot/' to your gdb command." fi