From d7553ed000a269c0cbdebcc2a121a957141fc29a Mon Sep 17 00:00:00 2001 From: Andrew Bradshaw Date: Thu, 13 Feb 2025 11:56:16 -0800 Subject: [PATCH] SERVER-96986 Add missing install targets (#32410) GitOrigin-RevId: 6ff09be543382cc892241a3a077b85ba54480a42 --- .bazelrc | 2 + .gitignore | 1 + BUILD.bazel | 71 +++++++++++++ bazel/config/BUILD.bazel | 17 +++ bazel/config/configs.bzl | 11 ++ bazel/install_rules/install_rules.bzl | 2 +- bazel/install_rules/install_rules.py | 16 ++- bazel/wrapper_hook/autogenerated_targets.py | 108 ++++++++++++++++++++ bazel/wrapper_hook/target_compatable.cquery | 6 ++ bazel/wrapper_hook/wrapper_hook.py | 5 + 10 files changed, 234 insertions(+), 5 deletions(-) create mode 100644 bazel/wrapper_hook/autogenerated_targets.py create mode 100644 bazel/wrapper_hook/target_compatable.cquery diff --git a/.bazelrc b/.bazelrc index 30ee69433b0..36e6cf006bd 100644 --- a/.bazelrc +++ b/.bazelrc @@ -98,6 +98,8 @@ common --flag_alias=enterprise_feature_all=//bazel/config:enterprise_feature_all common --flag_alias=link_timeout=//bazel/config:link_timeout common --flag_alias=compress_debug_compile=//bazel/config:compress_debug_compile common --flag_alias=include_mongot=//bazel/config:include_mongot +# This flag will only work if you pass it on the command line as it is parsed by bazelisk pre-bazel script +common --flag_alias=include_autogenerated_targets=//bazel/config:include_autogenerated_targets common --flag_alias=dwarf_version=//bazel/config:dwarf_version common --flag_alias=http_client=//bazel/config:http_client diff --git a/.gitignore b/.gitignore index feea918d159..dd736baea90 100644 --- a/.gitignore +++ b/.gitignore @@ -289,6 +289,7 @@ buildozer *.bazel_info_for_ninja.txt .ninja_last_command_line_targets.txt bazel/coverity/analysis/BUILD.bazel +src/mongo/db/modules/enterprise/autogenerated_targets/BUILD.bazel .bazel_include_info.json .bazel_header_list_cache .bazel_real diff --git a/BUILD.bazel b/BUILD.bazel index afb25981a86..6f033e52855 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -182,3 +182,74 @@ mongo_install( "//src/mongo/db/modules/enterprise:dist-test", ], ) + +mongo_install( + name = "mongo-crypt", + srcs = [ + "//src/mongo/db:service_context_non_d", + "//src/mongo/db:service_context_test_fixture", + "//src/mongo/util/options_parser", + ], + deps = [ + "//src/mongo/db/modules/enterprise:mongo-crypt", + ], +) + +mongo_install( + name = "dbtest", + srcs = [ + "//src/mongo/dbtests:dbtest", + ], +) + +mongo_install( + name = "sdam_json_test", + srcs = [ + "//src/mongo/client/sdam:sdam_json_test", + ], +) + +mongo_install( + name = "server_selection_json_test", + srcs = [ + "//src/mongo/client/sdam:server_selection_json_test", + ], +) + +mongo_install( + name = "stitch_support_dev", + srcs = [ + "//src/mongo/embedded/stitch_support", + ], +) + +[ + mongo_install( + name = target_name, + testonly = True, + srcs = select({ + "//bazel/config:include_autogenerated_targets_enabled": ["//src/mongo/db/modules/enterprise/autogenerated_targets:" + target_name], + "//conditions:default": [], + }), + ) + for target_name in [ + "bsoncolumn_bm", + "mongo_benchmark", + "mongo_fuzzer_test", + "mongo_integration_test", + "mongo_unittest", + "mongo_unittest_first_group", + "mongo_unittest_second_group", + "mongo_unittest_third_group", + "mongo_unittest_fourth_group", + "mongo_unittest_fifth_group", + "mongo_unittest_sixth_group", + "mongo_unittest_seventh_group", + "mongo_unittest_eighth_group", + "query_bm", + "repl_bm", + "sharding_bm", + "sep_bm", + "storage_bm", + ] +] diff --git a/bazel/config/BUILD.bazel b/bazel/config/BUILD.bazel index 988cbb8ed07..b78a29f283d 100644 --- a/bazel/config/BUILD.bazel +++ b/bazel/config/BUILD.bazel @@ -19,6 +19,7 @@ load( "fsan", "gcov", "http_client", + "include_autogenerated_targets", "include_mongot", "js_engine", "libunwind", @@ -963,6 +964,22 @@ config_setting( }, ) +# -------------------------------------- +# Include autogenerated target options +# -------------------------------------- + +include_autogenerated_targets( + name = "include_autogenerated_targets", + build_setting_default = False, +) + +config_setting( + name = "include_autogenerated_targets_enabled", + flag_values = { + "//bazel/config:include_autogenerated_targets": "True", + }, +) + # -------------------------------------- # allocator options # -------------------------------------- diff --git a/bazel/config/configs.bzl b/bazel/config/configs.bzl index 1922e1beb66..5de1c0c8fcb 100644 --- a/bazel/config/configs.bzl +++ b/bazel/config/configs.bzl @@ -108,6 +108,17 @@ include_mongot = rule( build_setting = config.bool(flag = True), ) +# ========= +# include_autogenerated_targets +# ========= + +include_autogenerated_targets_provider = provider(doc = "Enable using autogenerated build targets.", fields = ["enabled"]) + +include_autogenerated_targets = rule( + implementation = lambda ctx: include_autogenerated_targets_provider(enabled = ctx.build_setting_value), + build_setting = config.bool(flag = True), +) + # ========= # allocator # ========= diff --git a/bazel/install_rules/install_rules.bzl b/bazel/install_rules/install_rules.bzl index b54e60363f0..0c102ad29e0 100644 --- a/bazel/install_rules/install_rules.bzl +++ b/bazel/install_rules/install_rules.bzl @@ -166,7 +166,7 @@ def mongo_install_rule_impl(ctx): libs = [lib for lib in file_map["dynamic_libs"]] + [lib for lib in file_map["dynamic_libs_debug"]] unittest_bin = None - if len(bins) == 1 and file_map["binaries"][bins[0]].basename.endswith("_test"): + if len(bins) == 1 and ctx.attr.debug != "debug" and file_map["binaries"][bins[0]].basename.endswith("_test"): unittest_bin = file_map["binaries"][bins[0]] # create a dep file for passing all the files we intend to install diff --git a/bazel/install_rules/install_rules.py b/bazel/install_rules/install_rules.py index 67a9a231433..6aea6c83afa 100644 --- a/bazel/install_rules/install_rules.py +++ b/bazel/install_rules/install_rules.py @@ -4,20 +4,28 @@ import os import shutil parser = argparse.ArgumentParser() + parser.add_argument("--depfile", action="append") parser.add_argument("--install-dir") parser.add_argument("--install-mode", choices=["copy", "symlink", "hardlink"], default="hardlink") args = parser.parse_args() - if os.path.exists(args.install_dir): os.chmod(args.install_dir, 0o755) for root, dirs, files in os.walk(args.install_dir): for name in files: - os.chmod(os.path.join(root, name), 0o755) - os.unlink(os.path.join(root, name)) + try: + os.chmod(os.path.join(root, name), 0o755) + os.unlink(os.path.join(root, name)) + # Sometimes we find files that don't exist + # from os.walk - not sure why + except FileNotFoundError: + continue for name in dirs: - os.chmod(os.path.join(root, name), 0o755) + try: + os.chmod(os.path.join(root, name), 0o755) + except FileNotFoundError: + continue shutil.rmtree(args.install_dir) os.makedirs(args.install_dir, exist_ok=True) diff --git a/bazel/wrapper_hook/autogenerated_targets.py b/bazel/wrapper_hook/autogenerated_targets.py new file mode 100644 index 00000000000..047a6d6937b --- /dev/null +++ b/bazel/wrapper_hook/autogenerated_targets.py @@ -0,0 +1,108 @@ +import os +import subprocess + +bazel_tags_to_autogenerate = [ + "bsoncolumn_bm", + "mongo_benchmark", + "mongo_fuzzer_test", + "mongo_integration_test", + "mongo_unittest", + "mongo_unittest_first_group", + "mongo_unittest_second_group", + "mongo_unittest_third_group", + "mongo_unittest_fourth_group", + "mongo_unittest_fifth_group", + "mongo_unittest_sixth_group", + "mongo_unittest_seventh_group", + "mongo_unittest_eighth_group", + "query_bm", + "repl_bm", + "sharding_bm", + "sep_bm", + "storage_bm", +] + + +def autogenerate_targets(args, bazel): + bazel_autogenerate_flag = "include_autogenerated_targets" + output_file = os.path.join( + "src", "mongo", "db", "modules", "enterprise", "autogenerated_targets", "BUILD.bazel" + ) + if not any(bazel_autogenerate_flag in arg for arg in args): + if os.path.exists(output_file): + os.remove(output_file) + return + generate_targets(args, bazel, output_file) + + +def generate_targets( + args=[], + bazel="bazel", + output_file=os.path.join( + "src", "mongo", "db", "modules", "enterprise", "autogenerated_targets", "BUILD.bazel" + ), +): + targets = [] + for tag in bazel_tags_to_autogenerate: + labels = get_bazel_labels_from_tags(args, bazel, tag) + filtered_labels = filter_labels(labels) + targets.append(create_target(tag, filtered_labels)) + write_bazel_file(targets, output_file) + + +def get_bazel_labels_from_tags(args, bazel, tag): + # This way we mostly keep the passed config, this parsing isn't perfect + # for the ways you can call bazel configs but this should mostly just be used in CI + # where we control how things are called + extra_args = [arg for arg in args if arg.startswith("--")] + # The .cquery file is used to get info on which targets are compatible + # with our current config. Without it dependent targets would just be skipped. + proc = subprocess.run( + [bazel, "cquery", "--config=local"] + + extra_args + + [ + f"kind(extract_debuginfo_test, attr(tags, '\\b{tag}\\b', //src/...))", + "--output=starlark", + "--starlark:file=bazel/wrapper_hook/target_compatable.cquery", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + return proc.stdout.splitlines() + + +# We need to filter out any labels who do not get built in our current configuration +def filter_labels(labels): + filtered_labels = set() + for label in labels: + label_name, can_build = label.split(" ") + if can_build == "True": + # Labels get passed with an @ sign + filtered_labels.add(label_name[1:]) + return filtered_labels + + +def create_target(tag, labels): + target = [] + target.append("filegroup(") + target.append(f' name="{tag}",') + target.append(" srcs=[") + for label in labels: + target.append(f' "{label}",') + target.append(" ],") + target.append(" testonly=True,") + target.append(' visibility=["//visibility:public"],') + target.append(")") + return "\n".join(target) + + +# We default write the enterprise directory so we can access both bazel package targets +def write_bazel_file(targets, output_file): + with open(output_file, "w") as buildfile: + for target in targets: + buildfile.write(target + "\n\n") + + +if __name__ == "__main__": + generate_targets() diff --git a/bazel/wrapper_hook/target_compatable.cquery b/bazel/wrapper_hook/target_compatable.cquery new file mode 100644 index 00000000000..300ae42f950 --- /dev/null +++ b/bazel/wrapper_hook/target_compatable.cquery @@ -0,0 +1,6 @@ +def format(target): + if "IncompatiblePlatformProvider" in providers(target): + result = "False" + else: + result = "True" + return "%s %s" % (target.label, result) diff --git a/bazel/wrapper_hook/wrapper_hook.py b/bazel/wrapper_hook/wrapper_hook.py index 1897664cccb..938ff630b01 100644 --- a/bazel/wrapper_hook/wrapper_hook.py +++ b/bazel/wrapper_hook/wrapper_hook.py @@ -18,6 +18,7 @@ wrapper_debug(f"wrapper hook script is using {sys.executable}") def main(): install_modules(sys.argv[1]) + from bazel.wrapper_hook.autogenerated_targets import autogenerate_targets from bazel.wrapper_hook.developer_bes_keywords import write_workstation_bazelrc from bazel.wrapper_hook.engflow_check import engflow_auth from bazel.wrapper_hook.plus_interface import test_runner_interface @@ -26,6 +27,10 @@ def main(): write_workstation_bazelrc(sys.argv) + # This is used to autogenerate a BUILD.bazel that creates + # Filegroups for select tags - used to group targets for installing + autogenerate_targets(sys.argv, sys.argv[1]) + args = test_runner_interface( sys.argv[1:], autocomplete_query=os.environ.get("MONGO_AUTOCOMPLETE_QUERY") == "1" )