From b4b16fa91f6789e93046b67c8538694edcdab6be Mon Sep 17 00:00:00 2001 From: Ryan Berryhill Date: Wed, 5 Feb 2025 11:11:05 -0500 Subject: [PATCH] SERVER-95382 Introduce python module to find the mongo toolchain (#31229) GitOrigin-RevId: cd137fb08af14a307c8c45fa56c908d50c2b8d1d --- bazel/mongo_script_rules.bzl | 89 ++ buildscripts/BUILD.bazel | 32 + buildscripts/clang_format.py | 7 +- buildscripts/clang_tidy.py | 11 +- buildscripts/clang_tidy_vscode.py | 8 +- buildscripts/mongo_toolchain.py | 201 ++++ buildscripts/tests/BUILD.bazel | 19 + buildscripts/tests/test_clang_tidy.py | 8 +- buildscripts/toolchains.py | 938 ------------------- etc/toolchains.yaml | 30 - src/mongo/tools/mongo_tidy_checks/SConscript | 26 +- 11 files changed, 381 insertions(+), 988 deletions(-) create mode 100644 bazel/mongo_script_rules.bzl create mode 100644 buildscripts/mongo_toolchain.py create mode 100644 buildscripts/tests/BUILD.bazel delete mode 100755 buildscripts/toolchains.py delete mode 100644 etc/toolchains.yaml diff --git a/bazel/mongo_script_rules.bzl b/bazel/mongo_script_rules.bzl new file mode 100644 index 00000000000..f4ed32b0fb7 --- /dev/null +++ b/bazel/mongo_script_rules.bzl @@ -0,0 +1,89 @@ +"""Common mongo-specific bazel build rules intended to be used for buildscripts. +""" + +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") + +MONGO_TOOLCHAIN_V4_PATH = "/opt/mongodbtoolchain/v4" +MONGO_TOOLCHAIN_V5_PATH = "external/mongo_toolchain_v5/v5" + +def _py_cxx_wrapper(*, python_path, toolchain_path, python_interpreter, main_py): + return "\n".join([ + "export PYTHONPATH={}".format(python_path), + "export MONGO_TOOLCHAIN_PATH={}".format(toolchain_path), + "{} {}".format(python_interpreter, main_py), + ]) + +def _py_cxx_test_impl(ctx): + python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime + + python_path = [] + for dep in ctx.attr.deps: + for path in dep[PyInfo].imports.to_list(): + if path not in python_path: + python_path.append( + ctx.expand_make_variables( + "python_library_imports", + "$${RUNFILES_DIR}/" + path, + ctx.var, + ), + ) + python_path_str = ctx.configuration.host_path_separator.join(python_path) + + cc_toolchain = find_cpp_toolchain(ctx) + runfiles = ctx.runfiles( + files = ( + ctx.files.srcs + + ctx.files.data + + ctx.files.deps + + ctx.files.main + + cc_toolchain.all_files.to_list() + ), + ) + transitive_runfiles = [] + for runfiles_attr in ( + [ctx.attr.main], + ctx.attr.srcs, + ctx.attr.deps, + ctx.attr.data, + ): + for target in runfiles_attr: + transitive_runfiles.append(target[DefaultInfo].default_runfiles) + runfiles = runfiles.merge_all(transitive_runfiles) + + main_py = ctx.attr.main.files.to_list()[0].path + script = _py_cxx_wrapper( + python_path = python_path_str, + toolchain_path = ctx.attr.toolchain_path, + python_interpreter = python.interpreter.path, + main_py = main_py, + ) + ctx.actions.write( + output = ctx.outputs.executable, + content = script, + ) + + return DefaultInfo(files = depset([ctx.outputs.executable]), runfiles = runfiles) + +py_cxx_test = rule( + implementation = _py_cxx_test_impl, + attrs = { + "main": attr.label(allow_single_file = True, mandatory = True), + "srcs": attr.label_list(allow_files = [".py"]), + "deps": attr.label_list(), + "data": attr.label_list(), + "toolchain_path": attr.string(mandatory = True), + }, + toolchains = ["@bazel_tools//tools/cpp:toolchain_type", "@bazel_tools//tools/python:toolchain_type"], + executable = True, + test = True, +) + +def mongo_toolchain_py_cxx_test(**kwargs): + py_cxx_test( + toolchain_path = select({ + "//bazel/config:mongo_toolchain_v5": MONGO_TOOLCHAIN_V5_PATH, + "//conditions:default": MONGO_TOOLCHAIN_V4_PATH, + }), + target_compatible_with = ["@//bazel/platforms:use_mongo_toolchain"], + **kwargs + ) diff --git a/buildscripts/BUILD.bazel b/buildscripts/BUILD.bazel index 96ae7247d19..df6bd886601 100644 --- a/buildscripts/BUILD.bazel +++ b/buildscripts/BUILD.bazel @@ -68,6 +68,10 @@ py_binary( "click", group = "evergreen", ), + dependency( + "typing-extensions", + group = "core", + ), ], ) @@ -194,3 +198,31 @@ sh_binary( }, visibility = ["//visibility:public"], ) + +py_library( + name = "mongo_toolchain", + srcs = [ + "mongo_toolchain.py", + ], + visibility = ["//visibility:public"], + deps = [ + dependency( + "typer", + group = "core", + ), + ], +) + +py_library( + name = "clang_tidy_lib", + srcs = [ + "apply_clang_tidy_fixes.py", + "clang_tidy.py", + "clang_tidy_vscode.py", + ], + visibility = ["//visibility:public"], + deps = [ + "mongo_toolchain", + "simple_report", + ], +) diff --git a/buildscripts/clang_format.py b/buildscripts/clang_format.py index 02452db2ca7..bbd8e1c8755 100755 --- a/buildscripts/clang_format.py +++ b/buildscripts/clang_format.py @@ -32,6 +32,7 @@ if __name__ == "__main__" and __package__ is None: from buildscripts.linter import git, parallel from buildscripts.linter.filediff import gather_changed_files_for_lint +from buildscripts.mongo_toolchain import get_mongo_toolchain ############################################################################## # @@ -47,12 +48,14 @@ CLANG_FORMAT_SHORTER_VERSION = "120" # Name of clang-format as a binary CLANG_FORMAT_PROGNAME = "clang-format" +TOOLCHAIN_VERSION = "v4" + CLANG_FORMAT_HTTP_DARWIN_CACHE = ( "http://mongodbtoolchain.build.10gen.cc/toolchain/osx/clang-format-12.0.1" ) -# TODO: Move clang format to the v4 toolchain -CLANG_FORMAT_TOOLCHAIN_PATH = "/opt/mongodbtoolchain/v4/bin/clang-format" +toolchain = get_mongo_toolchain(version=TOOLCHAIN_VERSION) +CLANG_FORMAT_TOOLCHAIN_PATH = toolchain.get_tool_path(CLANG_FORMAT_PROGNAME) ############################################################################## diff --git a/buildscripts/clang_tidy.py b/buildscripts/clang_tidy.py index 35ee44a0bbd..34aa3129e87 100755 --- a/buildscripts/clang_tidy.py +++ b/buildscripts/clang_tidy.py @@ -18,7 +18,12 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple import yaml + +# Get relative imports to work when the package is not installed on the PYTHONPATH. +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + from clang_tidy_vscode import CHECKS_SO +from mongo_toolchain import get_mongo_toolchain from simple_report import make_report, put_report, try_combine_reports checks_so = "" @@ -163,7 +168,8 @@ def __dedup_errors(clang_tidy_errors_threads: List[str]) -> str: def _run_tidy(args, parser_defaults): - clang_tidy_binary = f"/opt/mongodbtoolchain/{args.clang_tidy_toolchain}/bin/clang-tidy" + toolchain = get_mongo_toolchain(version=args.clang_tidy_toolchain) + clang_tidy_binary = toolchain.get_tool_path("clang-tidy") if os.path.exists(args.check_module): mongo_tidy_check_module = args.check_module @@ -366,8 +372,7 @@ def main(): default=False, help="if this is a test evaluating clang tidy itself.", ) - # TODO: Is there someway to get this without hardcoding this much - parser.add_argument("-y", "--clang-tidy-toolchain", type=str, default="v4") + parser.add_argument("-y", "--clang-tidy-toolchain", type=str, default=None) parser.add_argument("-f", "--clang-tidy-cfg", type=str, default=config_file) args = parser.parse_args() diff --git a/buildscripts/clang_tidy_vscode.py b/buildscripts/clang_tidy_vscode.py index fcab914a2d3..63596ec40be 100755 --- a/buildscripts/clang_tidy_vscode.py +++ b/buildscripts/clang_tidy_vscode.py @@ -28,6 +28,11 @@ import os import subprocess import sys +# Get relative imports to work when the package is not installed on the PYTHONPATH. +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +from mongo_toolchain import get_mongo_toolchain + CHECKS_SO = [ "build/install/lib/libmongo_tidy_checks.so", ] @@ -38,7 +43,8 @@ if os.path.exists(".mongo_checks_module_path"): def main(): - clang_tidy_args = ["/opt/mongodbtoolchain/v4/bin/clang-tidy"] + toolchain = get_mongo_toolchain() + clang_tidy_args = [toolchain.get_tool_path("clang-tidy")] for check_lib in CHECKS_SO: if os.path.isfile(check_lib): clang_tidy_args += [f"-load={check_lib}"] diff --git a/buildscripts/mongo_toolchain.py b/buildscripts/mongo_toolchain.py new file mode 100644 index 00000000000..678fa69dec0 --- /dev/null +++ b/buildscripts/mongo_toolchain.py @@ -0,0 +1,201 @@ +import os +import subprocess +import sys +from pathlib import Path +from typing import Optional + +import typer +from typing_extensions import Annotated + +# Get relative imports to work when the package is not installed on the PYTHONPATH. +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__))))) + + +SUPPORTED_VERSIONS = ("v4", "v5") + + +class MongoToolchainError(RuntimeError): + pass + + +class MongoToolchainNotFoundError(MongoToolchainError): + def __init__(self, extra_msg: str): + super().__init__(extra_msg) + + +class ToolNotFoundError(MongoToolchainError): + def __init__(self, tool: str, extra_msg: str | None): + msg = f"Couldn't find {tool} in mongo toolchain" + msg = msg + f": {extra_msg}" if extra_msg is not None else msg + super().__init__(msg) + + +class MongoToolchain: + def __init__(self, root_dir_path: Path): + self._root_dir = root_dir_path + + def get_tool_path(self, tool: str) -> str: + path = self._get_bin_dir_path() / tool + if not path.exists(): + raise ToolNotFoundError(tool, f"{path} does not exist") + if not path.is_file() or not os.access(path, os.X_OK): + raise ToolNotFoundError(tool, f"{path} is not an executable") + return str(path) + + def check_exists(self) -> None: + for directory in ( + self._root_dir, + self._get_bin_dir_path(), + self._get_include_dir_path(), + self._get_lib_dir_path(), + ): + if not directory.is_dir(): + raise MongoToolchainNotFoundError(f"{directory} is not a directory") + + def get_root_dir(self) -> str: + return str(self._root_dir) + + def get_bin_dir(self) -> str: + return str(self._get_bin_dir_path()) + + def get_include_dir(self) -> str: + return str(self._get_include_dir_path()) + + def get_lib_dir(self) -> str: + return str(self._get_include_dir_path()) + + def _get_bin_dir_path(self) -> Path: + return self._root_dir / "bin" + + def _get_include_dir_path(self) -> Path: + return self._root_dir / "include" + + def _get_lib_dir_path(self) -> Path: + return self._root_dir / "lib" + + +def _run_command(cmd: str) -> str: + return subprocess.check_output(cmd, shell=True, text=True).strip() + + +def _fetch_bazel_toolchain(version: str) -> None: + try: + _run_command(f"bazel build @mongo_toolchain_{version}//:all") + except subprocess.CalledProcessError as e: + raise MongoToolchainNotFoundError( + f"Failed to fetch bazel toolchain: `{e.cmd}` exited with code {e.returncode}" + ) + + +def _get_bazel_execroot() -> Path: + try: + execroot_str = _run_command("bazel info execution_root") + except subprocess.CalledProcessError as e: + raise MongoToolchainNotFoundError( + f"Couldn't find bazel execroot: `{e.cmd}` exited with code {e.returncode}" + ) + execroot = Path(execroot_str) + return execroot + + +def _get_bazel_toolchain_path(version: str) -> Path: + return _get_bazel_execroot() / "external" / f"mongo_toolchain_{version}" / version + + +def _get_toolchain_from_path(path: str) -> MongoToolchain: + toolchain = MongoToolchain(Path(path).resolve()) + toolchain.check_exists() + return toolchain + + +def _get_bazel_toolchain(version: str) -> MongoToolchain: + path = _get_bazel_toolchain_path(version) + if not path.exists(): + _fetch_bazel_toolchain(version) + if not path.is_dir(): + raise MongoToolchainNotFoundError( + f"Couldn't find bazel toolchain: {path} is not a directory" + ) + return _get_toolchain_from_path(path) + + +def _get_installed_toolchain_path(version: str) -> Path: + return Path("/opt/mongodbtoolchain") / version + + +def _get_installed_toolchain(version: str): + return _get_toolchain_from_path(_get_installed_toolchain_path(version)) + + +def get_mongo_toolchain( + *, version: str | None = None, from_bazel: bool | None = None +) -> MongoToolchain: + # When running under bazel this environment variable will be set and will point to the + # toolchain the target was configured to use. It can also be set manually to override + # a script's selection of toolchain. + toolchain_path = os.environ.get("MONGO_TOOLCHAIN_PATH", None) + if toolchain_path is not None: + return _get_toolchain_from_path(toolchain_path) + + # If no version given, look in the environment or default to v4. + if version is None: + version = os.environ.get("MONGO_TOOLCHAIN_VERSION", "v4") + assert version is not None + if version not in SUPPORTED_VERSIONS: + raise MongoToolchainNotFoundError(f"Unknown toolchain version {version}") + + # If from_bazel is unspecified. Look in the environment or fall back to a default based on + # the version (v4 -> not from bazel, v5 -> from bazel). + def _parse_from_bazel_envvar(value: str) -> bool: + v = value.lower() + if v in ("true", "1"): + return True + elif v in ("false", "0"): + return False + else: + raise ValueError(f"Invalid value {value} for MONGO_TOOLCHAIN_FROM_BAZEL") + + if from_bazel is None: + from_bazel_default = "false" if version == "v4" else "true" + from_bazel_value = os.environ.get("MONGO_TOOLCHAIN_FROM_BAZEL", from_bazel_default) + from_bazel = _parse_from_bazel_envvar(from_bazel_value) + + if from_bazel: + return _get_bazel_toolchain(version) + return _get_installed_toolchain(version) + + +def try_get_mongo_toolchain(*args, **kwargs) -> MongoToolchain | None: + try: + return get_mongo_toolchain(*args, **kwargs) + except MongoToolchainError: + return None + + +_app = typer.Typer(add_completion=False) + + +@_app.command() +def main( + tool: Annotated[Optional[str], typer.Argument()] = None, + version: Annotated[Optional[str], typer.Option("--version")] = None, + from_bazel: Annotated[Optional[bool], typer.Option("--bazel/--no-bazel")] = None, +): + """ + Prints the path to tools in the mongo toolchain or the toolchain's root directory (which + should contain bin/, include/, and so on). + If MONGO_TOOLCHAIN_PATH is set in the environment, it overrides all options to this script. + Otherwise, MONGO_TOOLCHAIN_VERSION is a lower-precedence alternative to --version and + MONGO_TOOLCHAIN_FROM_BAZEL is a lower-precedence alternative to --bazel/--no-bazel. + None of these are required, and if none are given, the default configuration will be used. + """ + toolchain = get_mongo_toolchain(version=version, from_bazel=from_bazel) + if tool is not None: + print(toolchain.get_tool_path(tool)) + else: + print(toolchain.get_root_dir()) + + +if __name__ == "__main__": + _app() diff --git a/buildscripts/tests/BUILD.bazel b/buildscripts/tests/BUILD.bazel new file mode 100644 index 00000000000..1ced62d2086 --- /dev/null +++ b/buildscripts/tests/BUILD.bazel @@ -0,0 +1,19 @@ +load("@poetry//:dependencies.bzl", "dependency") +load("//bazel:mongo_script_rules.bzl", "mongo_toolchain_py_cxx_test") + +mongo_toolchain_py_cxx_test( + name = "test_clang_tidy", + srcs = [ + "test_clang_tidy.py", + ], + main = "test_clang_tidy.py", + visibility = ["//visibility:public"], + deps = [ + "//buildscripts:clang_tidy_lib", + "//buildscripts:mongo_toolchain", + dependency( + "pyyaml", + group = "core", + ), + ], +) diff --git a/buildscripts/tests/test_clang_tidy.py b/buildscripts/tests/test_clang_tidy.py index dae60fe7a4a..76d1fdfcdfd 100644 --- a/buildscripts/tests/test_clang_tidy.py +++ b/buildscripts/tests/test_clang_tidy.py @@ -11,6 +11,7 @@ import yaml sys.path.append("buildscripts") import apply_clang_tidy_fixes from clang_tidy import _clang_tidy_executor, _combine_errors +from mongo_toolchain import get_mongo_toolchain @unittest.skipIf( @@ -42,7 +43,8 @@ void f(std::string s); self.fixes_dir = os.path.join(self.tempdir.name, "fixes") os.mkdir(self.fixes_dir) - self.clang_tidy_binary = "/opt/mongodbtoolchain/v4/bin/clang-tidy" + toolchain = get_mongo_toolchain() + self.clang_tidy_binary = toolchain.get_tool_path("clang-tidy") clang_tidy_cfg = "Checks: 'performance-unnecessary-value-param'" self.clang_tidy_cfg = yaml.safe_load(clang_tidy_cfg) @@ -103,3 +105,7 @@ void f(std::string s); header.read(), "The clang-tidy fix should not have been applied.", ) + + +if __name__ == "__main__": + unittest.main() diff --git a/buildscripts/toolchains.py b/buildscripts/toolchains.py deleted file mode 100755 index eec2639c8dd..00000000000 --- a/buildscripts/toolchains.py +++ /dev/null @@ -1,938 +0,0 @@ -#!/usr/bin/env python3 -"""Provides data regarding mongodbtoolchain.""" - -import argparse -import enum -import os -import pathlib -import string -import sys -import warnings -from typing import ( - Any, - Callable, - Dict, - Generator, - Iterator, - List, - Mapping, - Optional, - Sequence, - Set, - Tuple, - Union, -) - -import yaml - -__all__ = [ - "DEFAULT_DATA_FILE", - "Toolchain", - "ToolchainConfig", - "ToolchainDataException", - "ToolchainReleaseName", - "ToolchainVersionName", - "Toolchains", -] - -DEFAULT_DATA_FILE: pathlib.Path = pathlib.Path(__file__).parent / "../etc/toolchains.yaml" - - -class ToolchainVersionName(str, enum.Enum): - """Represents a "named" toolchain version, such as "stable" or "testing".""" - - STABLE = "stable" - TESTING = "testing" - - # pylint: disable=invalid-str-returned - def __str__(self) -> str: - return self.value - - -class ToolchainReleaseName(str, enum.Enum): - """Represents a "named" toolchain release, such as "rollback" or "current".""" - - ROLLBACK = "rollback" - CURRENT = "current" - LATEST = "latest" - - # pylint: disable=invalid-str-returned - def __str__(self) -> str: - return self.value - - -class ToolchainDistroName(Tuple[str, ...], enum.Enum): - """Represents a distribution for which the toolchain is built.""" - - AMAZON1_2012 = ( - "amazon1-2012", - "linux-64-amzn", - ) - AMAZON1_2018 = ("amazon1-2018",) - AMAZON2 = ("amazon2",) - ARCHLINUX = ("archlinux",) - CENTOS6 = ("centos6",) - DEBIAN8 = ("debian81",) - DEBIAN9 = ("debian92",) - DEBIAN10 = ("debian10",) - DEBIAN11 = ("debian11",) - DEBIAN12 = ("debian12",) - MACOS1012 = ("macos-1012",) - MACOS1014 = ("macos-1014",) - MACOS1100 = ("macos-1100",) - RHEL6 = ("rhel6", "rhel62", "rhel67") - RHEL7 = ("rhel7", "rhel70", "rhel71", "rhel72", "rhel76", "ubi7") - RHEL8 = ("rhel8", "rhel80", "rhel81", "rhel82", "rhel83", "rhel84", "rhel88", "ubi8") - SUSE12 = ("suse12", "suse12-sp5") - SUSE15 = ("suse15", "suse15-sp0", "suse15-sp2") - UBUNTU1404 = ("ubuntu1404",) - UBUNTU1604 = ("ubuntu1604",) - UBUNTU1804 = ("ubuntu1804",) - UBUNTU2004 = ("ubuntu2004",) - DEFAULT = ("default",) - - @classmethod - def from_str(cls, text: str) -> "ToolchainDistroName": - """Return the enumeration object matching a given string.""" - - for distro in cls: - if text in distro.value: - return distro - - raise ValueError(f"Unknown distro string: {text}") - - # pylint: disable=unsubscriptable-object - def __str__(self) -> str: - return self.value[0] - - -class ToolchainArchName(Tuple[str, ...], enum.Enum): - """Represents an architecture for which the toolchain is built.""" - - ARM64 = ("arm64", "aarch64") - PPC64LE = ("ppc64le", "power8") - S390X = ("s390x", "zSeries") - X86_64 = ("x86_64",) - DEFAULT = ("",) - - @classmethod - def from_str(cls, text: str) -> "ToolchainArchName": - """Return the enumeratrion object matching a given string.""" - - for arch in cls: - if text in arch.value: - return arch - - raise ValueError(f"Unknown arch string: {text}") - - # pylint: disable=unsubscriptable-object - def __str__(self) -> str: - return self.value[0] - - -class ToolchainDataException(Exception): - """Represents a problem encountered while reading or querying toolchain data.""" - - -class ToolchainPlatform: - """Represents a platform for which the toolchain is built.""" - - def __init__(self, distro_id: str, arch: Optional[ToolchainArchName] = None) -> None: - """Parse a distro_id into a full toolchain platform.""" - - self._distro_id: str = distro_id - - self._distro: Optional[ToolchainDistroName] = None - self._arch: Optional[ToolchainArchName] = arch - self._tag: Optional[str] = None - - # These two actions are order-dependent! - self._distro_length: int = self._find_distro_length() - self._arch_span: Tuple[int, int] = self._find_arch_span() - - def _split_distro_id(self, start: int = 0) -> Tuple[str, str]: - return self._distro_id[start:].split("-", 1)[0], self._distro_id[start:].split(".")[0] - - def _find_distro_length(self) -> int: - for distro in ToolchainDistroName: - for name in distro.value: - if not name: - continue - - if name.lower() in self._split_distro_id(): - return len(name) - - raise ValueError(f"Failed to match distro from distro_id: `{self._distro_id}'") - - def _find_arch_span(self) -> Tuple[int, int]: - arch_span: Optional[Tuple[int, int]] = None - - for arch in ToolchainArchName: - for name in arch.value: - if not name: - continue - - iter_start = self._distro_length + 1 - while iter_start < len(self._distro_id): - if name.lower() in self._split_distro_id(self._distro_length): - arch_span = (iter_start, len(name)) - - iter_start += len(name) - if iter_start < len(self._distro_id) and self._distro_id[iter_start] in ( - "-", - ".", - ): - iter_start += 1 - - if arch_span is None: - arch_span = (self._distro_length, 0) - - return arch_span - - @property - def distro_id(self) -> str: - """Return the distro_id.""" - - return self._distro_id - - @property - def distro(self) -> ToolchainDistroName: - """Return the distro component of the distro_id.""" - - if self._distro is None: - distro_length: int = self._distro_length - self._distro = ToolchainDistroName.from_str(self._distro_id[:distro_length]) - - return self._distro - - @property - def arch(self) -> ToolchainArchName: - """Return the architecture component of the distro_id.""" - - if self._arch is None: - arch_span: Tuple[int, int] = self._arch_span - if arch_span[1] == 0: - # There is no architecture specified in the distro_id - if self.distro == ToolchainDistroName.DEFAULT: - # The "default" distro is allowed to have a nonexistent arch - self._arch = ToolchainArchName.DEFAULT - else: - self._arch = ToolchainArchName.X86_64 - else: - self._arch = ToolchainArchName.from_str( - self.distro_id[arch_span[0] : arch_span[0] + arch_span[1]] - ) - - return self._arch - - @property - def tag(self) -> Optional[str]: - """Return the descriptive tag component of the distro_id.""" - - if self._tag is None: - arch_span: Tuple[int, int] = self._arch_span - if arch_span[0] + arch_span[1] + 1 < len(self._distro_id): - self._tag = self._distro_id[arch_span[0] + arch_span[1] + 1 :] - else: - self._tag = "" - - if self._tag: - return self._tag - - return None - - def __str__(self) -> str: - result: str = f"{self.distro}" - - if self.arch != ToolchainArchName.DEFAULT: - result += f".{self.arch}" - - if self.tag is not None: - result += f".{self.tag}" - - return result - - -class ToolchainConfig: - """Represents a toolchain configuration.""" - - def __init__(self, data_file: pathlib.Path, platform: ToolchainPlatform) -> None: - """Construct a toolchain configuration from a data file.""" - - try: - with open(data_file.absolute(), "r", encoding="utf-8") as yaml_stream: - self._data = yaml.safe_load(yaml_stream) - except yaml.YAMLError as parent_exc: - raise ToolchainDataException( - f"Could not read toolchain data file: `{data_file}'" - ) from parent_exc - - self._platform: ToolchainPlatform = platform - - @property - def base_path(self) -> pathlib.Path: - """Return the base (installed) path for toolchain releases.""" - - return pathlib.Path(self._data["toolchains"]["base_path"]) - - @property - def all_releases(self) -> Dict[str, Dict[str, str]]: - """Return all known releases in the data file.""" - - try: - return self._data["toolchains"]["releases"] - except (KeyError, TypeError): - return {} - - @property - def releases(self) -> Dict[str, str]: - """Return all releases for a the current platform.""" - - # Successively match the distro and/or platform against available toolchains. - # If none is found, use the default entry. If the default entry doesn't exist, - # we want to know about it because that's a misconfiguration. - platform_section: Optional[Dict[str, str]] = None - - if self._platform.distro_id in self.all_releases: - platform_section = self.all_releases[self._platform.distro_id] - elif self._platform.distro != ToolchainDistroName.DEFAULT: - if str(self._platform) in self.all_releases: - platform_section = self.all_releases[str(self._platform)] - elif f"{self._platform.distro}.{self._platform.arch}" in self.all_releases: - platform_section = self.all_releases[ - f"{self._platform.distro}.{self._platform.arch}" - ] - elif f"{self._platform.distro}.{self._platform.tag}" in self.all_releases: - platform_section = self.all_releases[ - f"{self._platform.distro}.{self._platform.tag}" - ] - elif f"{self._platform.distro}" in self.all_releases: - platform_section = self.all_releases[f"{self._platform.distro}"] - - if not platform_section: - try: - platform_section = self.all_releases["default"] - except KeyError: - return {} - - return platform_section - - @property - def versions(self) -> List[str]: - """Return all known versions in the data file.""" - - return self._data["toolchains"]["versions"] - - @property - def aliases(self) -> Dict[str, str]: - """Return all known version aliases in the data file.""" - - return self._data["toolchains"]["version_aliases"] - - @property - def revisions_dir(self) -> Optional[pathlib.Path]: - """Return the legacy revisions directory for toolchain releases.""" - - warnings.warn( - ( - "This is legacy toolchain usage. " - f"Call {self.__class__.__name__}.releases_dir() instead." - ), - DeprecationWarning, - stacklevel=2, - ) - - return self.base_path.joinpath("revisions") - - @property - def releases_dir(self) -> pathlib.Path: - """Return the directory where toolchain releases are installed.""" - - return self.base_path.joinpath("releases") - - def search_releases(self, release_name: str) -> Optional[str]: - """Search configured releases for a given release name.""" - - try: - return self.releases[release_name] - except KeyError: - return None - - -class Toolchain: - """Represents the raw toolchain data.""" - - def __init__(self, config: ToolchainConfig, release: str) -> None: - """Construct a toolchain object from a supplied release and version.""" - - self._config = config - self._release = release - - @property - def release(self) -> str: - """Return the toolchain release ID.""" - - return self._release - - @property - def install_path(self) -> pathlib.Path: - """Return the path to where the toolchain should be installed.""" - - path = self._config.releases_dir / self.release - - # We need to determine if the configured release is a legacy release - # that only appears in revisions_dir. We can tell the difference - # because legacy releases are all identified by git commit hashes. - if len(self.release) == 40 and set(self.release) <= set(string.hexdigits): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - if self._config.revisions_dir: - path = self._config.revisions_dir / self.release - - return path - - def exec_path(self, version: Union[ToolchainVersionName, str]) -> Optional[pathlib.Path]: - """Return a path to a specific toolchain version.""" - - install_path = self.install_path - - if isinstance(version, ToolchainVersionName): - version = version.value - - if version not in self._config.versions: - try: - version = self._config.aliases[version] - except KeyError: - raise ValueError( - f"Toolchain version `{version}' not defined in data file" - ) from None - - return install_path / version - - -class Toolchains(Mapping[Union[ToolchainReleaseName, str], Toolchain]): - """Represents a collection of toolchains that may or may not be installed on a system.""" - - def __init__(self, config: ToolchainConfig) -> None: - """Manipulate raw toolchain configuration data.""" - - self._config = config - self._available: Optional[List[str]] = None - - def _search_filesystem(self) -> List[str]: - release_dirs: List[pathlib.Path] = [] - - releases_dir: Optional[pathlib.Path] = self._config.releases_dir - if releases_dir and releases_dir.exists(): - release_dirs.extend([path for path in releases_dir.iterdir() if path.is_dir()]) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - revisions_dir: Optional[pathlib.Path] = self._config.revisions_dir - - if revisions_dir and revisions_dir.exists(): - release_dirs.extend([path for path in revisions_dir.iterdir() if path.is_dir()]) - - if release_dirs: - return [ - path.name - for path in sorted( - release_dirs, key=lambda path: path.stat().st_mtime, reverse=True - ) - ] - - return [] - - @property - def available(self) -> List[str]: - """Return a list of all installed toolchain releases ordered from newest to oldest.""" - - if self._available is None: - self._available = self._search_filesystem() - - return self._available - - @property - def configured(self) -> List[str]: - """Return a list of all configured toolchain releases.""" - - configured: Set[Union[str, None]] = { - self._config.search_releases(name.value) for name in ToolchainReleaseName - } - - configured.add(self._config.search_releases("default")) - - return [release for release in configured if release is not None] - - @property - def latest(self) -> Optional[str]: - """Return the latest installed toolchain release. - - This can possibly be different from available[0] - because we take into account the "latest" symlink present on end-user toolchain - installations, which could potentially not be the "newest" for any reason. Therefore, these - methods can be used to determine whether the "latest" symlink is out of date. - """ - - latest_symlink: Optional[pathlib.Path] = None - try: - latest_symlink = self._config.releases_dir.joinpath("latest") - except AttributeError: - latest_symlink = None - - if latest_symlink and latest_symlink.exists(): - return pathlib.Path(os.readlink(latest_symlink.absolute())).name - - return self.available[0] or None - - def __iter__(self) -> Iterator[str]: - for release in set(self.available).union(set(self.configured)): - yield release - - def __len__(self) -> int: - return len(list(self.__iter__())) - - def __getitem__(self, key: Union[ToolchainReleaseName, str]) -> Toolchain: - """Return the named toolchain release. - - This method supports two disjoint use cases: - - 1. We don't know the release ID and pass a release name. Here, we are attempting to - determine the release ID corresponding to the name for the current platform. If no - such release name is configured, we raise an exception to indicate a potential - misconfiguration or code error. - - 2. We know the release ID and want to determine whether it is installed. In this case, - None is returned if the release is not available to indicate it is not installed. - """ - - release: Optional[str] = None - - if isinstance(key, ToolchainReleaseName): - # We don't know the release ID and are querying by name. This supports use case 1. - if key == ToolchainReleaseName.LATEST: - latest = self.latest - if latest is None: - raise KeyError(key) - - release = self.latest - - release = self._config.search_releases(str(key)) - else: - # We know the release ID and want to know whether it's installed. This supports use - # case 2. This is the "short path" for the method because any string other than a - # real release ID or release name causes a return. - if key == str(ToolchainReleaseName.LATEST): - release = self.latest - if key in self.available: - release = key - if key in [name.value for name in ToolchainReleaseName]: - release = self._config.search_releases(key) - - if release is None: - raise KeyError(key) - - return Toolchain(self._config, release) - - -class _FormatterClass: - """A protocol for a formatter class. - - HelpFormatter is missing this protocol in its inheritance tree in - some versions of the mypy typeshed, which causes a spurious mypy error - when trying to assign a custom HelpFormatter subclass. This is just - here for NicerHelpFormatter to inherit from it and prevent the error. - """ - - def __call__(self, _: str) -> argparse.HelpFormatter: ... - - -# pylint: disable=protected-access -class NicerHelpFormatter(argparse.HelpFormatter, _FormatterClass): - """A HelpFormatter with nicer output than the default.""" - - def __init__( - self, prog: str, indent_increment: int = 2, max_help_position: int = 32, width=None - ) -> None: - super().__init__( - prog=prog, - indent_increment=indent_increment, - max_help_position=max_help_position, - width=width, - ) - - def __call__(self, prog: str) -> argparse.HelpFormatter: - return NicerHelpFormatter(prog) - - def _format_action(self, action: argparse.Action) -> str: - result = super()._format_action(action) - if isinstance(action, (argparse._SubParsersAction, DictChoiceAction)): - return (" " * self._current_indent) + f"{result.lstrip()}" - return result - - def _format_action_invocation(self, action: argparse.Action) -> str: - if isinstance(action, (argparse._SubParsersAction, DictChoiceAction)): - return "" - if not action.option_strings: - default = self._get_default_metavar_for_optional(action) - (metavar,) = self._metavar_formatter(action, default)(1) - return metavar - - parts: List[str] = [] - - if action.nargs == 0: - parts.extend(action.option_strings) - else: - default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) - - for option_string in action.option_strings: - parts.append(option_string) - - return f"{' '.join(parts)} {args_string}" - - return " ".join(parts) - - def _iter_indented_subactions( - self, action: argparse.Action - ) -> Generator[argparse.Action, None, None]: - if isinstance(action, (argparse._SubParsersAction, DictChoiceAction)): - try: - get_subactions = action._get_subactions - except AttributeError: - pass - else: - for subaction in get_subactions(): - yield subaction - else: - for subaction in super()._iter_indented_subactions(action): - yield subaction - - def _metavar_formatter( - self, action: argparse.Action, default_metavar: str - ) -> Callable[[int], Tuple[str, ...]]: - if action.metavar is not None: - result = action.metavar - elif action.choices is not None: - choice_strs = [f"{choice}" for choice in action.choices] - result = f"{' | '.join(choice_strs)}" - if action.required: - result = f"({result})" - else: - result = f"[{result}]" - else: - result = default_metavar - - def _format(tuple_size: int) -> Tuple[str, ...]: - if isinstance(result, tuple): - return result - return (result,) * tuple_size - - return _format - - -# pylint: disable=protected-access, redefined-builtin, redefined-outer-name -class DictChoiceAction(argparse._StoreAction): - """An action with nicer per-choice formatting.""" - - class _ChoicesPseudoAction(argparse.Action): - def __init__(self, name: str, aliases: List[str], help: Optional[str] = None) -> None: - metavar = dest = name - if aliases: - metavar += f" {' | '.join(aliases)}" - if self.required: - metavar = f"({metavar})" - else: - metavar = f"[{metavar})" - - super().__init__(option_strings=[], dest=dest, help=help, metavar=metavar) - - def __call__( - self, - parser: argparse.ArgumentParser, - namespace: argparse.Namespace, - values: Union[str, Sequence[Any], None], - option_string: Optional[str] = None, - ) -> None: - parser.print_help() - parser.exit() - - def __init__( - self, - option_strings: List[str], - dest: str, - nargs: Optional[int] = None, - const: Optional[Any] = None, - default: Optional[Any] = None, - type: Optional[type] = None, - choices: Optional[Dict[str, str]] = None, - required: bool = False, - help: Optional[str] = None, - metavar: Optional[str] = None, - ) -> None: - super().__init__( - option_strings=option_strings, - dest=dest, - nargs=nargs, - const=const, - default=default, - type=type, - choices=choices, - required=required, - help=help, - metavar=metavar, - ) - self.choices: Dict[str, str] = {} - if choices: - self.choices = choices - - def _get_subactions(self): - choices_actions: List[argparse.Action] = [] - for name, help_text in self.choices.items(): - choice_action = self._ChoicesPseudoAction(name, [], help_text) - choices_actions.append(choice_action) - return choices_actions - - -class NicerArgumentParser(argparse.ArgumentParser): - """An argument parser with nicer help output.""" - - def __init__( - self, - prog: Optional[str] = None, - usage: Optional[str] = None, - description: Optional[str] = None, - epilog: Optional[str] = None, - parents: Optional[List[argparse.ArgumentParser]] = None, - prefix_chars: str = "-", - fromfile_prefix_chars: Optional[str] = None, - argument_default: Optional[Any] = None, - conflict_handler: str = "error", - add_help: bool = True, - allow_abbrev: bool = True, - ) -> None: - """Initialize a NicerParser.""" - - super().__init__( - prog=prog, - usage=usage, - description=description, - epilog=epilog, - parents=parents if parents else [], - formatter_class=NicerHelpFormatter, - prefix_chars=prefix_chars, - fromfile_prefix_chars=fromfile_prefix_chars, - argument_default=argument_default, - conflict_handler=conflict_handler, - add_help=add_help, - allow_abbrev=allow_abbrev, - ) - self._optionals.title = "Options" - self._positionals.title = "Queries" - - def format_help(self) -> str: - formatter = self._get_formatter() - formatter.add_text(self.description) - formatter.add_usage( - self.usage, self._actions, self._mutually_exclusive_groups, prefix="Usage:\n " - ) - - for action_group in self._action_groups: - formatter.start_section(action_group.title) - formatter.add_text(action_group.description) - formatter.add_arguments(action_group._group_actions) - formatter.end_section() - - formatter.add_text(self.epilog) - - return formatter.format_help() - - -if __name__ == "__main__": - parser = NicerArgumentParser( - description="Tool for querying information about mongodbtoolchain.", add_help=False - ) - - parser.add_argument( - "-h", - "--help", - action="help", - default=argparse.SUPPRESS, - help="Show this help message and exit", - ) - parser.add_argument( - "-f", - "--from-file", - help="Specify a toolchain data file", - metavar="FILE", - type=str, - default=str(DEFAULT_DATA_FILE), - ) - parser.add_argument("-d", "--distro-id", help="Evergreen distro_id", type=str, required=True) - parser.add_argument("-a", "--arch", help="Host architecture", type=str) - - subparsers = parser.add_subparsers(title="Commands", dest="command", required=True) - - show_parser = subparsers.add_parser( - "show", - description="Shows general toolchain collection info.", - add_help=False, - help="Show general toolchain collection info", - ) - config_parser = subparsers.add_parser( - "config", - description="Shows toolchain configuration info.", - add_help=False, - help="Show toolchain configuration info", - ) - platform_parser = subparsers.add_parser( - "platform", - description="Shows component parts of a distro_id.", - add_help=False, - help="Show parts of a distro_id", - ) - toolchain_parser = subparsers.add_parser( - "toolchain", - description="Shows specific toolchain info.", - add_help=False, - help="Show specific toolchain info", - ) - - show_parser.add_argument( - "-h", - "--help", - action="help", - default=argparse.SUPPRESS, - help="Show this help message and exit", - ) - show_parser.add_argument( - "query", - action=DictChoiceAction, - type=str, - choices={ - "available": "All installed toolchains", - "configured": "Toolchains configured for the distro_id", - "latest": "The most recent installed toolchain", - }, - ) - - config_parser.add_argument( - "-h", - "--help", - action="help", - default=argparse.SUPPRESS, - help="Show this help message and exit", - ) - config_parser.add_argument( - "query", - action=DictChoiceAction, - type=str, - choices={ - "base_path": "Toolchain base execution path", - "releases": "All defined release names", - "versions": "All defined version names", - "aliases": "All defined aliases for version names", - }, - ) - - platform_parser.add_argument( - "-h", - "--help", - action="help", - default=argparse.SUPPRESS, - help="Show this help message and exit", - ) - platform_parser.add_argument( - "query", - action=DictChoiceAction, - type=str, - choices={ - "distro": 'Show the "distro" component of the distro_id', - "arch": 'Show the "arch" component of the distro_id', - "tag": "Show the information tag component of the distro_id", - }, - ) - - toolchain_parser.add_argument( - "-h", - "--help", - action="help", - default=argparse.SUPPRESS, - help="Show this help message and exit", - ) - toolchain_parser.add_argument( - "-v", - "--toolchain-version", - help="Toolchain version", - type=str, - default=str(ToolchainVersionName.STABLE), - ) - toolchain_parser.add_argument( - "-r", - "--release", - help="Toolchain release", - type=str, - default=str(ToolchainReleaseName.CURRENT), - ) - toolchain_parser.add_argument( - "query", - action=DictChoiceAction, - type=str, - choices={ - "install_path": "Toolchain installation path", - "exec_path": "Toolchain execution path", - }, - ) - - parsed_args = parser.parse_args() - obj: Optional[object] = None - - # Set up the objects required for each command - toolchain_platform = ToolchainPlatform(distro_id=parsed_args.distro_id, arch=parsed_args.arch) - if parsed_args.command == "platform": - obj = toolchain_platform - elif parsed_args.command in ("show", "config", "toolchain"): - try: - toolchain_config = ToolchainConfig( - pathlib.Path(parsed_args.from_file), platform=toolchain_platform - ) - except ToolchainDataException as exc: - print(exc, file=sys.stderr) - sys.exit(1) - - toolchains = Toolchains(config=toolchain_config) - - if parsed_args.command == "show": - obj = toolchains - elif parsed_args.command == "config": - obj = toolchain_config - elif parsed_args.command == "toolchain": - obj = toolchains[parsed_args.release] - else: - print(f"Unknown command: {parsed_args.command}", file=sys.stderr) - sys.exit(1) - - # Get and handle output - output: Any - attribute = getattr(obj, parsed_args.query) - if callable(attribute): - if attribute.__name__ == "exec_path": - output = attribute(parsed_args.toolchain_version) - else: - output = attribute() - else: - output = attribute # If there is no output, it should indicate an error to the caller. - - # pylint: disable=invalid-name - if output is not None: - if isinstance(output, (tuple, list)): - output = str.join(" ", output) - elif isinstance(output, dict): - output = "\n".join([f"{k}: {v}" for k, v in output.items()]) - elif not isinstance(output, str): - output = str(output) - - if output: - print(output) - else: - sys.exit(1) diff --git a/etc/toolchains.yaml b/etc/toolchains.yaml deleted file mode 100644 index 64bd4582747..00000000000 --- a/etc/toolchains.yaml +++ /dev/null @@ -1,30 +0,0 @@ -toolchains: - base_path: /opt/mongodbtoolchain - versions: - - v2 - - v3 - - v4 - version_aliases: - stable: v4 - testing: v4 - releases: - default: - current: 549e9c72ce95de436fb83815796d54a47893c049 - rhel80-s390x-playbook: - current: 41a191fa8daae48ef6271805fe30c004c69e18d4 - ubuntu1804-power8-playbook: - current: 8391ec859b03d3bebb666821a88e616ae61a567a - ubuntu1804-zseries-playbook: - current: 32eb70c47bd9e9759dd05654843feb80461aaef3 - archlinux-packer: - current: ca1de958b3e011a973bfdd2cb947360dab348502 - ubuntu1404-jepsen: - current: ca1de958b3e011a973bfdd2cb947360dab348502 - ubuntu1604-container: - current: ca1de958b3e011a973bfdd2cb947360dab348502 - suse12: - current: ca1de958b3e011a973bfdd2cb947360dab348502 - amazon1-2018: - current: ca1de958b3e011a973bfdd2cb947360dab348502 - macos-1012: - current: ca1de958b3e011a973bfdd2cb947360dab348502 diff --git a/src/mongo/tools/mongo_tidy_checks/SConscript b/src/mongo/tools/mongo_tidy_checks/SConscript index 88e760848b3..3e71915647c 100644 --- a/src/mongo/tools/mongo_tidy_checks/SConscript +++ b/src/mongo/tools/mongo_tidy_checks/SConscript @@ -1,32 +1,32 @@ Import("env") Import("use_libunwind") -from buildscripts.toolchains import DEFAULT_DATA_FILE, ToolchainConfig, ToolchainPlatform +from pathlib import Path + +from buildscripts.mongo_toolchain import try_get_mongo_toolchain toolchain_clang_tidy_dev_found = False toolchain_found = False base_toolchain_bin = None -try: - toolchain_config = ToolchainConfig(DEFAULT_DATA_FILE, ToolchainPlatform("default")) - toolchain_found = toolchain_config.base_path.exists() - base_toolchain_bin = toolchain_config.base_path / toolchain_config.aliases["stable"] / "bin" - base_revision_path = (base_toolchain_bin / "clang-tidy").resolve().parent.parent +toolchain = try_get_mongo_toolchain( + version=env.get("MONGO_TOOLCHAIN_VERSION", "v4"), from_bazel=False +) +toolchain_found = toolchain is not None +if not toolchain_found: + Return() +base_toolchain_bin = Path(toolchain.get_bin_dir()) - tidy_include = base_revision_path / "include" - tidy_lib = base_revision_path / "lib" +tidy_include = Path(toolchain.get_include_dir()) +tidy_lib = toolchain.get_lib_dir() - toolchain_clang_tidy_dev_found = (tidy_include / "clang-tidy" / "ClangTidy.h").exists() +toolchain_clang_tidy_dev_found = (tidy_include / "clang-tidy" / "ClangTidy.h").exists() -except FileNotFoundError: - pass if toolchain_found and not toolchain_clang_tidy_dev_found: # If there was a toolchain but its not setup right, issue a warning about this. print( "Could not find not find clang-tidy headers in toolchain, not building mongo custom checks module." ) - -if not toolchain_found or not toolchain_clang_tidy_dev_found: Return() env = env.Clone()