SERVER-96986 Added MongoT support to Bazel Compile Evergreen Task (#32210)

GitOrigin-RevId: 2bc6a12fb659a135dda15351b2277f70d4e80818

.bazelrc

@@ -97,6 +97,7 @@ common --flag_alias=use_sasl_client=//bazel/config:use_sasl_client
 common --flag_alias=enterprise_feature_all=//bazel/config:enterprise_feature_all
 common --flag_alias=link_timeout=//bazel/config:link_timeout
 common --flag_alias=compress_debug_compile=//bazel/config:compress_debug_compile
+common --flag_alias=include_mongot=//bazel/config:include_mongot
 
 common --flag_alias=dwarf_version=//bazel/config:dwarf_version
 common --flag_alias=http_client=//bazel/config:http_client

BUILD.bazel

@@ -4,6 +4,7 @@ load("//bazel/toolchains:mongo_toolchain.bzl", "setup_mongo_toolchain_aliases")
 load("//bazel/config:render_template.bzl", "render_template")
 load("@npm//:eslint/package_json.bzl", eslint_bin = "bin")
 load("@aspect_rules_js//js:defs.bzl", "js_library")
+load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -142,6 +143,17 @@ mongo_install(
     ],
 )
 
+copy_to_directory(
+    name = "mongot_folder",
+    srcs = ["@mongot_localdev//:mongot_binaries"],
+    out = "mongot-localdev",
+    include_external_repositories = ["mongot_localdev"],
+    tags = [
+        "local",
+        "no-cache",
+    ],
+)
+
 mongo_install(
     name = "dist-test",
     srcs = [
@@ -159,6 +171,9 @@ mongo_install(
     ] + select({
         "@platforms//os:windows": ["@windows_sasl//:bins"],
         "//conditions:default": [],
+    }) + select({
+        "//bazel/config:include_mongot_enabled": ["//:mongot_folder"],
+        "//conditions:default": [],
     }),
     deps = [
         "//src/mongo/db/modules/enterprise:dist-test",

WORKSPACE.bazel

@@ -46,9 +46,9 @@ http_archive(
 # aspect_rules_lint depends on aspect_bazel_lib.
 http_archive(
     name = "aspect_bazel_lib",
-    sha256 = "6d758a8f646ecee7a3e294fbe4386daafbe0e5966723009c290d493f227c390b",
-    strip_prefix = "bazel-lib-2.7.7",
-    url = "https://github.com/aspect-build/bazel-lib/releases/download/v2.7.7/bazel-lib-v2.7.7.tar.gz",
+    sha256 = "57a777c5d4d0b79ad675995ee20fc1d6d2514a1ef3000d98f5c70cf0c09458a3",
+    strip_prefix = "bazel-lib-2.13.0",
+    url = "https://github.com/bazel-contrib/bazel-lib/releases/download/v2.13.0/bazel-lib-v2.13.0.tar.gz",
 )
 
 load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies")
@@ -353,3 +353,16 @@ setup_pigz(
 load("//bazel/format:shfmt.bzl", "shfmt")
 
 shfmt()
+
+# This repository is normally created by db-contrib-tool or manually extracting the binaries at the proper location
+new_local_repository(
+    name = "mongot_localdev",
+    build_file_content = """
+package(default_visibility = ["//visibility:public"])
+filegroup(
+    name = "mongot_binaries",
+    srcs = glob(["**"], exclude = ["BUILD.bazel", "WORKSPACE"]),
+)
+""",
+    path = "mongot-localdev",
+)

bazel/config/BUILD.bazel

@@ -19,6 +19,7 @@ load(
     "fsan",
     "gcov",
     "http_client",
+    "include_mongot",
     "js_engine",
     "libunwind",
     "linker",
@@ -946,6 +947,22 @@ config_setting(
     },
 )
 
+# --------------------------------------
+# Include MongoT options
+# --------------------------------------
+
+include_mongot(
+    name = "include_mongot",
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "include_mongot_enabled",
+    flag_values = {
+        "//bazel/config:include_mongot": "True",
+    },
+)
+
 # --------------------------------------
 # allocator options
 # --------------------------------------

bazel/config/configs.bzl

@@ -97,6 +97,17 @@ spider_monkey_dbg = rule(
     build_setting = config.bool(flag = True),
 )
 
+# =========
+# include_mongot
+# =========
+
+include_mongot_provider = provider(doc = "Enable including mongot in the final binary archive from a local mongot-localdev folder.", fields = ["enabled"])
+
+include_mongot = rule(
+    implementation = lambda ctx: include_mongot_provider(enabled = ctx.build_setting_value),
+    build_setting = config.bool(flag = True),
+)
+
 # =========
 # allocator
 # =========

bazel/install_rules/install_rules.bzl

@@ -71,7 +71,23 @@ def check_debug(ctx, basename):
         ctx.fail("Unknown OS")
         return False
 
-def sort_file(ctx, file, install_dir, file_map):
+def declare_output(ctx, output, is_directory):
+    """Declare an output as either a file or directory
+
+    Args:
+        ctx: rule ctx
+        output: output file to declare
+        is_directory: determines if the file is a directory
+
+    Returns:
+        File object representing output
+    """
+    if is_directory:
+        return ctx.actions.declare_directory(output)
+    else:
+        return ctx.actions.declare_file(output)
+
+def sort_file(ctx, file, install_dir, file_map, is_directory):
     """Determine location a file should be installed to
 
     Args:
@@ -79,6 +95,7 @@ def sort_file(ctx, file, install_dir, file_map):
         file: file to sort
         install_dir: the directory to install to
         file_map: dict containing specific file designations
+        is_directory: determines if the file is a directory
 
     """
     _, macos_constraint, _ = get_constraints(ctx)
@@ -89,22 +106,16 @@ def sort_file(ctx, file, install_dir, file_map):
     if check_binary(ctx, basename):
         if not check_debug(ctx, basename):
             if ctx.attr.debug != "debug":
-                file_map["binaries"][file] = ctx.actions.declare_file(bin_install)
+                file_map["binaries"][file] = declare_output(ctx, bin_install, is_directory)
         elif ctx.attr.debug != "stripped":
-            if ctx.target_platform_has_constraint(macos_constraint):
-                file_map["binaries_debug"][file] = ctx.actions.declare_directory(bin_install)
-            else:
-                file_map["binaries_debug"][file] = ctx.actions.declare_file(bin_install)
+            file_map["binaries_debug"][file] = declare_output(ctx, bin_install, is_directory)
 
     elif not check_debug(ctx, basename):
         if ctx.attr.debug != "debug":
-            file_map["dynamic_libs"][file] = ctx.actions.declare_file(lib_install)
+            file_map["dynamic_libs"][file] = declare_output(ctx, lib_install, is_directory)
 
     elif ctx.attr.debug != "stripped":
-        if ctx.target_platform_has_constraint(macos_constraint):
-            file_map["dynamic_libs_debug"][file] = ctx.actions.declare_directory(lib_install)
-        else:
-            file_map["dynamic_libs_debug"][file] = ctx.actions.declare_file(lib_install)
+        file_map["dynamic_libs_debug"][file] = declare_output(ctx, lib_install, is_directory)
 
 def mongo_install_rule_impl(ctx):
     """Perform install actions
@@ -130,16 +141,18 @@ def mongo_install_rule_impl(ctx):
     # sort direct sources
     for input_bin in ctx.attr.srcs:
         for bin in input_bin.files.to_list():
-            sort_file(ctx, bin.path, install_dir, file_map)
+            sort_file(ctx, bin.path, install_dir, file_map, bin.is_directory)
 
     # sort dependency install files
     for dep in ctx.attr.deps:
+        # Create a map of filename to if its a directory, ie. { coolfolder: True, coolfile: False } as the json loses that info
+        file_directory_map = {file_dep.basename: file_dep.is_directory for file_dep in dep[DefaultInfo].files.to_list()}
         src_map = json.decode(dep[MongoInstallInfo].src_map.to_list()[0])
         files = []
         for key in src_map:
             files.extend(src_map[key])
         for file in files:
-            sort_file(ctx, file, install_dir, file_map)
+            sort_file(ctx, file, install_dir, file_map, file_directory_map[file.split("/")[-1]])
 
     # aggregate based on type of installs
     if ctx.attr.debug == "stripped":

bazel/install_rules/install_rules.py

@@ -56,7 +56,8 @@ def install(src, install_type):
                                 dest_dir = os.path.dirname(os.path.join(root, name)).replace(
                                     src, dst
                                 )
-                                os.makedirs(dest_dir)
+                                if not os.path.exists(dest_dir):
+                                    os.makedirs(dest_dir)
                                 os.link(os.path.join(root, name), os.path.join(dest_dir, name))
                     else:
                         os.link(src, dst)

evergreen/bazel_compile.sh

@@ -16,6 +16,79 @@ cd src
 set -o errexit
 set -o verbose
 
+activate_venv
+
+# if build_patch_id is passed, try to download binaries from specified
+# evergreen patch.
+build_patch_id="${build_patch_id:-${reuse_compile_from}}"
+if [ -n "${build_patch_id}" ]; then
+  echo "build_patch_id detected, trying to skip task"
+  if [ "${task_name}" = "compile_dist_test" ] || [ "${task_name}" = "compile_dist_test_half" ]; then
+    echo "Skipping ${task_name} compile without downloading any files"
+    exit 0
+  fi
+
+  # On windows we change the extension to zip
+  if [ -z "${ext}" ]; then
+    ext="tgz"
+  fi
+
+  extra_db_contrib_args=""
+
+  # get the platform of the dist archive. This is needed if
+  # db-contrib-tool cannot autodetect the platform of the ec2 instance.
+  regex='MONGO_DISTMOD=([a-z0-9]*)'
+  if [[ ${compile_flags} =~ ${regex} ]]; then
+    extra_db_contrib_args="${extra_db_contrib_args} --platform=${BASH_REMATCH[1]}"
+  fi
+
+  download_dir="./tmp_db_contrib_tool_download_dir"
+  rm -rf ${download_dir}
+
+  if [ "${task_name}" = "compile_dist_test" ]; then
+    file_name="mongodb-binaries.${ext}"
+    invocation="db-contrib-tool setup-repro-env ${build_patch_id} \
+      --variant=${compile_variant} --extractDownloads=False \
+      --binariesName=${file_name} --installDir=${download_dir} ${extra_db_contrib_args}"
+  fi
+
+  if [ "${task_name}" = "archive_dist_test_debug" ]; then
+    file_name="mongo-debugsymbols.${ext}"
+    invocation="db-contrib-tool setup-repro-env ${build_patch_id} \
+      --variant=${compile_variant} --extractDownloads=False \
+      --debugsymbolsName=${file_name} --installDir=${download_dir} \
+      --skipBinaries --downloadSymbols ${extra_db_contrib_args}"
+  fi
+
+  if [ -n "${invocation}" ]; then
+    setup_db_contrib_tool
+
+    echo "db-contrib-tool invocation: ${invocation}"
+    eval ${invocation}
+    if [ $? -ne 0 ]; then
+      echo "Could not retrieve files with db-contrib-tool"
+      exit 1
+    fi
+    file_location=$(find "${download_dir}" -name "${file_name}")
+    echo "Downloaded: ${file_location}"
+    mv "${file_location}" "${file_name}"
+    echo "Moved ${file_name} to the correct location"
+    echo "Skipping ${task_name} compile"
+    exit 0
+  fi
+
+  echo "Could not skip ${task_name} compile, compiling as normal"
+fi
+
+# --build-mongot is a compile flag used by the evergreen build variants that run end-to-end search
+# suites, as it downloads the necessary mongot binary.
+if [ "${build_mongot}" = "true" ]; then
+  setup_db_contrib_tool
+  use_db_contrib_tool_mongot
+fi
+
+set -o pipefail
+
 # Use `eval` to force evaluation of the environment variables in the echo statement:
 eval echo "Execution environment: Targets: ${targets}"
 

evergreen/prelude_db_contrib_tool.sh

@@ -17,3 +17,56 @@ function setup_db_contrib_tool {
 
   timeout_and_retry 180 python -m pipx install -vv --force "db-contrib-tool==0.8.5" --pip-args="--no-cache-dir"
 }
+
+function use_db_contrib_tool_mongot {
+  # Checking that this is not a downstream patch on mongod created by mongot's patch trigger.
+  # In the case that it's not, download latest (eg HEAD of 10gen/mongot) or the
+  # release (eg currently running in production on Atlas) mongot binary.
+  arch=$(uname -i)
+  if [[ ! $(declare -p linux_x86_64_mongot_localdev_binary linux_aarch64_mongot_localdev_binary macos_x86_64_mongot_localdev_binary 2> /dev/null) ]]; then
+
+    if [ "${download_mongot_release}" = "true" ]; then
+      mongot_version="release"
+    else
+      mongot_version="latest"
+    fi
+
+    if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+      mongot_platform="linux"
+    elif [[ "$OSTYPE" == "darwin"* ]]; then
+      mongot_platform="macos"
+    else
+      echo "mongot is only supported on linux and mac and does not support ${OSTYPE}"
+      exit 1
+    fi
+
+    mongot_arch="x86_64"
+    # macos arm64 is not supported by mongot, but macos x86_64 runs on it successfully
+    if [[ $arch == "aarch64"* ]] && [[ "$OSTYPE" != "darwin"* ]]; then
+      mongot_arch="aarch64"
+    fi
+    echo "running: db-contrib-tool setup-mongot-repro-env ${mongot_version} --platform=${mongot_platform} --architecture=${mongot_arch} --installDir=."
+    # This should create the folder mongot-localdev, usually run at the root of mongo directory
+    db-contrib-tool setup-mongot-repro-env ${mongot_version} --platform=${mongot_platform} --architecture=${mongot_arch} --installDir=.
+  else
+    # This is a downstream patch, which means there is a patched mongot binary we need to install.
+    if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+      if [[ $arch == "x86_64"* ]]; then
+        mongot_url=${linux_x86_64_mongot_localdev_binary}
+      elif [[ $arch == "aarch64"* ]]; then
+        mongot_url=${linux_aarch64_mongot_localdev_binary}
+      else
+        echo "mongot-localdev does not support ${arch}"
+        exit 1
+      fi
+    elif [[ "$OSTYPE" == "darwin"* ]]; then
+      mongot_url=${macos_x86_64_mongot_localdev_binary}
+    else
+      echo "mongot-localdev does not support ${OSTYPE}"
+      exit 1
+    fi
+    echo "running curl ${mongot_url} | tar xvz"
+    # This should create the folder mongot-localdev, usually run at the root of mongo directory
+    curl ${mongot_url} | tar xvz
+  fi
+}