SERVER-96986 Revert "Revert "SERVER-96986 Add missing install targets (#32410)" (#32467)

GitOrigin-RevId: e9bb2a420dd59cd7d909fd01b1e19fb0a51f5834

.bazelrc

@@ -98,6 +98,8 @@ common --flag_alias=enterprise_feature_all=//bazel/config:enterprise_feature_all
 common --flag_alias=link_timeout=//bazel/config:link_timeout
 common --flag_alias=compress_debug_compile=//bazel/config:compress_debug_compile
 common --flag_alias=include_mongot=//bazel/config:include_mongot
+# This flag will only work if you pass it on the command line as it is parsed by bazelisk pre-bazel script
+common --flag_alias=include_autogenerated_targets=//bazel/config:include_autogenerated_targets
 
 common --flag_alias=dwarf_version=//bazel/config:dwarf_version
 common --flag_alias=http_client=//bazel/config:http_client

.gitignore

@@ -289,6 +289,7 @@ buildozer
 *.bazel_info_for_ninja.txt
 .ninja_last_command_line_targets.txt
 bazel/coverity/analysis/BUILD.bazel
+src/mongo/db/modules/enterprise/autogenerated_targets/BUILD.bazel
 .bazel_include_info.json
 .bazel_header_list_cache
 .bazel_real

BUILD.bazel

@@ -182,3 +182,74 @@ mongo_install(
         "//src/mongo/db/modules/enterprise:dist-test",
     ],
 )
+
+mongo_install(
+    name = "mongo-crypt",
+    srcs = [
+        "//src/mongo/db:service_context_non_d",
+        "//src/mongo/db:service_context_test_fixture",
+        "//src/mongo/util/options_parser",
+    ],
+    deps = [
+        "//src/mongo/db/modules/enterprise:mongo-crypt",
+    ],
+)
+
+mongo_install(
+    name = "dbtest",
+    srcs = [
+        "//src/mongo/dbtests:dbtest",
+    ],
+)
+
+mongo_install(
+    name = "sdam_json_test",
+    srcs = [
+        "//src/mongo/client/sdam:sdam_json_test",
+    ],
+)
+
+mongo_install(
+    name = "server_selection_json_test",
+    srcs = [
+        "//src/mongo/client/sdam:server_selection_json_test",
+    ],
+)
+
+mongo_install(
+    name = "stitch_support_dev",
+    srcs = [
+        "//src/mongo/embedded/stitch_support",
+    ],
+)
+
+[
+    mongo_install(
+        name = target_name,
+        testonly = True,
+        srcs = select({
+            "//bazel/config:include_autogenerated_targets_enabled": ["//src/mongo/db/modules/enterprise/autogenerated_targets:" + target_name],
+            "//conditions:default": [],
+        }),
+    )
+    for target_name in [
+        "bsoncolumn_bm",
+        "mongo_benchmark",
+        "mongo_fuzzer_test",
+        "mongo_integration_test",
+        "mongo_unittest",
+        "mongo_unittest_first_group",
+        "mongo_unittest_second_group",
+        "mongo_unittest_third_group",
+        "mongo_unittest_fourth_group",
+        "mongo_unittest_fifth_group",
+        "mongo_unittest_sixth_group",
+        "mongo_unittest_seventh_group",
+        "mongo_unittest_eighth_group",
+        "query_bm",
+        "repl_bm",
+        "sharding_bm",
+        "sep_bm",
+        "storage_bm",
+    ]
+]

bazel/config/BUILD.bazel

@@ -19,6 +19,7 @@ load(
     "fsan",
     "gcov",
     "http_client",
+    "include_autogenerated_targets",
     "include_mongot",
     "js_engine",
     "libunwind",
@@ -963,6 +964,22 @@ config_setting(
     },
 )
 
+# --------------------------------------
+# Include autogenerated target options
+# --------------------------------------
+
+include_autogenerated_targets(
+    name = "include_autogenerated_targets",
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "include_autogenerated_targets_enabled",
+    flag_values = {
+        "//bazel/config:include_autogenerated_targets": "True",
+    },
+)
+
 # --------------------------------------
 # allocator options
 # --------------------------------------

bazel/config/configs.bzl

@@ -108,6 +108,17 @@ include_mongot = rule(
     build_setting = config.bool(flag = True),
 )
 
+# =========
+# include_autogenerated_targets
+# =========
+
+include_autogenerated_targets_provider = provider(doc = "Enable using autogenerated build targets.", fields = ["enabled"])
+
+include_autogenerated_targets = rule(
+    implementation = lambda ctx: include_autogenerated_targets_provider(enabled = ctx.build_setting_value),
+    build_setting = config.bool(flag = True),
+)
+
 # =========
 # allocator
 # =========

bazel/install_rules/install_rules.bzl

@@ -166,7 +166,7 @@ def mongo_install_rule_impl(ctx):
         libs = [lib for lib in file_map["dynamic_libs"]] + [lib for lib in file_map["dynamic_libs_debug"]]
 
     unittest_bin = None
-    if len(bins) == 1 and file_map["binaries"][bins[0]].basename.endswith("_test"):
+    if len(bins) == 1 and ctx.attr.debug != "debug" and file_map["binaries"][bins[0]].basename.endswith("_test"):
         unittest_bin = file_map["binaries"][bins[0]]
 
     # create a dep file for passing all the files we intend to install

bazel/install_rules/install_rules.py

@@ -4,20 +4,28 @@ import os
 import shutil
 
 parser = argparse.ArgumentParser()
+
 parser.add_argument("--depfile", action="append")
 parser.add_argument("--install-dir")
 parser.add_argument("--install-mode", choices=["copy", "symlink", "hardlink"], default="hardlink")
 
 args = parser.parse_args()
-
 if os.path.exists(args.install_dir):
     os.chmod(args.install_dir, 0o755)
     for root, dirs, files in os.walk(args.install_dir):
         for name in files:
-            os.chmod(os.path.join(root, name), 0o755)
+            try:
-            os.unlink(os.path.join(root, name))
+                os.chmod(os.path.join(root, name), 0o755)
+                os.unlink(os.path.join(root, name))
+            # Sometimes we find files that don't exist
+            # from os.walk - not sure why
+            except FileNotFoundError:
+                continue
         for name in dirs:
-            os.chmod(os.path.join(root, name), 0o755)
+            try:
+                os.chmod(os.path.join(root, name), 0o755)
+            except FileNotFoundError:
+                continue
     shutil.rmtree(args.install_dir)
 os.makedirs(args.install_dir, exist_ok=True)
 

bazel/wrapper_hook/autogenerated_targets.py

@@ -0,0 +1,108 @@
+import os
+import subprocess
+
+bazel_tags_to_autogenerate = [
+    "bsoncolumn_bm",
+    "mongo_benchmark",
+    "mongo_fuzzer_test",
+    "mongo_integration_test",
+    "mongo_unittest",
+    "mongo_unittest_first_group",
+    "mongo_unittest_second_group",
+    "mongo_unittest_third_group",
+    "mongo_unittest_fourth_group",
+    "mongo_unittest_fifth_group",
+    "mongo_unittest_sixth_group",
+    "mongo_unittest_seventh_group",
+    "mongo_unittest_eighth_group",
+    "query_bm",
+    "repl_bm",
+    "sharding_bm",
+    "sep_bm",
+    "storage_bm",
+]
+
+
+def autogenerate_targets(args, bazel):
+    bazel_autogenerate_flag = "include_autogenerated_targets"
+    output_file = os.path.join(
+        "src", "mongo", "db", "modules", "enterprise", "autogenerated_targets", "BUILD.bazel"
+    )
+    if not any(bazel_autogenerate_flag in arg for arg in args):
+        if os.path.exists(output_file):
+            os.remove(output_file)
+        return
+    generate_targets(args, bazel, output_file)
+
+
+def generate_targets(
+    args=[],
+    bazel="bazel",
+    output_file=os.path.join(
+        "src", "mongo", "db", "modules", "enterprise", "autogenerated_targets", "BUILD.bazel"
+    ),
+):
+    targets = []
+    for tag in bazel_tags_to_autogenerate:
+        labels = get_bazel_labels_from_tags(args, bazel, tag)
+        filtered_labels = filter_labels(labels)
+        targets.append(create_target(tag, filtered_labels))
+    write_bazel_file(targets, output_file)
+
+
+def get_bazel_labels_from_tags(args, bazel, tag):
+    # This way we mostly keep the passed config, this parsing isn't perfect
+    # for the ways you can call bazel configs but this should mostly just be used in CI
+    # where we control how things are called
+    extra_args = [arg for arg in args if arg.startswith("--")]
+    # The .cquery file is used to get info on which targets are compatible
+    # with our current config. Without it dependent targets would just be skipped.
+    proc = subprocess.run(
+        [bazel, "cquery", "--config=local"]
+        + extra_args
+        + [
+            f"kind(extract_debuginfo_test, attr(tags, '\\b{tag}\\b', //src/...))",
+            "--output=starlark",
+            "--starlark:file=bazel/wrapper_hook/target_compatable.cquery",
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    return proc.stdout.splitlines()
+
+
+# We need to filter out any labels who do not get built in our current configuration
+def filter_labels(labels):
+    filtered_labels = set()
+    for label in labels:
+        label_name, can_build = label.split(" ")
+        if can_build == "True":
+            # Labels get passed with an @ sign
+            filtered_labels.add(label_name[1:])
+    return filtered_labels
+
+
+def create_target(tag, labels):
+    target = []
+    target.append("filegroup(")
+    target.append(f'    name="{tag}",')
+    target.append("    srcs=[")
+    for label in labels:
+        target.append(f'        "{label}",')
+    target.append("    ],")
+    target.append("    testonly=True,")
+    target.append('    visibility=["//visibility:public"],')
+    target.append(")")
+    return "\n".join(target)
+
+
+# We default write the enterprise directory so we can access both bazel package targets
+def write_bazel_file(targets, output_file):
+    with open(output_file, "w") as buildfile:
+        for target in targets:
+            buildfile.write(target + "\n\n")
+
+
+if __name__ == "__main__":
+    generate_targets()

bazel/wrapper_hook/target_compatable.cquery

@@ -0,0 +1,6 @@
+def format(target):
+    if "IncompatiblePlatformProvider" in providers(target):
+        result = "False"
+    else:
+        result = "True"
+    return "%s %s" % (target.label, result)

bazel/wrapper_hook/wrapper_hook.py

@@ -18,6 +18,7 @@ wrapper_debug(f"wrapper hook script is using {sys.executable}")
 def main():
     install_modules(sys.argv[1])
 
+    from bazel.wrapper_hook.autogenerated_targets import autogenerate_targets
     from bazel.wrapper_hook.developer_bes_keywords import write_workstation_bazelrc
     from bazel.wrapper_hook.engflow_check import engflow_auth
     from bazel.wrapper_hook.plus_interface import test_runner_interface
@@ -26,6 +27,10 @@ def main():
 
     write_workstation_bazelrc(sys.argv)
 
+    # This is used to autogenerate a BUILD.bazel that creates
+    # Filegroups for select tags - used to group targets for installing
+    autogenerate_targets(sys.argv, sys.argv[1])
+
     args = test_runner_interface(
         sys.argv[1:], autocomplete_query=os.environ.get("MONGO_AUTOCOMPLETE_QUERY") == "1"
     )