SERVER-99662 Implmement building msi's in bazel (#31404)

GitOrigin-RevId: 3d661bda6d6e5e4c322d5194ce36333ae2442461

.bazelrc

@@ -400,6 +400,9 @@ common --define=MONGO_DISTMOD=""
 # Default if .git directory is not present
 common --define=GIT_COMMIT_HASH="nogitversion"
 
+# The version of Visual C++ to use
+common --define=MSVC_VERSION="14.3"
+
 # TODO(WT-12780): delete this once wiredtiger switches to /.bazelrc.evergreen.
 try-import %workspace%/.bazelrc.evergreen_engflow_creds
 

BUILD.bazel

@@ -139,3 +139,8 @@ mongo_install(
         "//src/mongo/db/modules/enterprise:dist-test",
     ],
 )
+
+mongo_install(
+    name = "compass",
+    srcs = ["//src/mongo/installer/compass:compass_files"],
+)

WORKSPACE.bazel

@@ -82,6 +82,53 @@ filegroup(
     url = "https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-2.1.28.zip",
 )
 
+http_archive(
+    name = "wix_toolset",
+    build_file_content = """
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "wix_binaries",
+    srcs = select({
+        "@platforms//os:windows": glob(["*"]),
+        "//conditions:default": [],
+    }),
+)
+
+filegroup(
+    name = "candle",
+    srcs = select({
+        "@platforms//os:windows": ["candle.exe"],
+        "//conditions:default": [],
+    }),
+    data = select({
+        "@platforms//os:windows": [":wix_binaries"],
+        "//conditions:default": [],
+    }),
+)
+
+filegroup(
+    name = "light",
+    srcs = select({
+        "@platforms//os:windows": ["light.exe"],
+        "//conditions:default": [],
+    }),
+    data = select({
+        "@platforms//os:windows": [":wix_binaries"],
+        "//conditions:default": [],
+    }),
+)
+    """,
+    sha256 = "6ac824e1642d6f7277d0ed7ea09411a508f6116ba6fae0aa5f2c7daa2ff43d31",
+    url = "https://github.com/wixtoolset/wix3/releases/download/wix3141rtm/wix314-binaries.zip",
+)
+
+load("//bazel/install_rules:windows_msvc.bzl", "windows_msvc")
+
+windows_msvc(
+    name = "local_windows_msvc",
+)
+
 load("//bazel/toolchains:python_toolchain.bzl", "setup_mongo_python_toolchains")
 
 [register_toolchains(toolchain) for toolchain in setup_mongo_python_toolchains()]

bazel/config/BUILD.bazel

@@ -2142,6 +2142,14 @@ selects.config_setting_group(
     ],
 )
 
+selects.config_setting_group(
+    name = "ssl_enabled_build_enterprise_disabled",
+    match_all = [
+        ":build_enterprise_disabled",
+        ":ssl_enabled",
+    ],
+)
+
 # --------------------------------------
 # thin_lto
 # --------------------------------------

bazel/install_rules/install_rules.bzl

@@ -45,7 +45,7 @@ def check_binary(ctx, basename):
     elif ctx.target_platform_has_constraint(macos_constraint):
         return not basename.startswith("lib")
     elif ctx.target_platform_has_constraint(windows_constraint):
-        return basename.endswith(".exe") or basename.endswith(".pdb") or basename.endswith(".dll")
+        return basename.endswith(".exe") or basename.endswith(".pdb") or basename.endswith(".dll") or basename.endswith(".ps1")
     else:
         ctx.fail("Unknown OS")
         return False

bazel/install_rules/msi.bzl

@@ -0,0 +1,174 @@
+# Build an msi using the wix toolset.
+# Building the msi involves running candle.exe -> light.exe -> msitrim.py
+def mongo_msi_impl(ctx):
+    candle_in = []
+    candle_out = []
+    candle_arguments = [ctx.attr._candle_wrapper_script.files.to_list()[0].path, ctx.executable._candle.path, "-wx"]
+
+    # pass in version variables
+    # eg. 8.1.0-alpha
+    mongo_version = ctx.attr.mongo_version
+    if mongo_version in ctx.var:
+        mongo_version = ctx.var[mongo_version]
+
+    # eg. 8.1
+    mongo_major_version = ".".join(mongo_version.split(".")[:2])
+    candle_arguments.append("-dMongoDBMajorVersion=" + mongo_major_version)
+
+    # eg. 8.1.0
+    mongo_no_revision_version = mongo_version.split("-")[0]
+    candle_arguments.append("-dMongoDBVersion=" + mongo_no_revision_version)
+
+    # pass in folder variables needed by wix
+    for var, label in ctx.attr.deps.items():
+        folder = ""
+        for file in label.files.to_list():
+            symlink = ctx.actions.declare_file(var + "/" + file.basename)
+            ctx.actions.symlink(output = symlink, target_file = file)
+            candle_in.append(symlink)
+            folder = symlink.dirname
+        candle_arguments.append("-d" + var + "=" + folder + "/")
+
+    # pass in string variables needed by wix
+    for var, value in ctx.attr.wix_vars.items():
+        candle_arguments.append("-d" + var + "=" + value)
+
+    # pass in custom action
+    for file in ctx.attr.custom_action.files.to_list():
+        if file.extension == "dll":
+            candle_in.append(file)
+            candle_arguments.append("-dCustomActionDll=" + file.path)
+
+    # pass in merge module if needed
+    msvc_version = ctx.attr.msvc_version
+    if msvc_version in ctx.var:
+        msvc_version = ctx.var[msvc_version]
+    msvc_version = "_VC" + msvc_version.replace(".", "") + "_"
+    if ctx.attr.use_merge_modules:
+        for file in ctx.attr._merge_modules.files.to_list():
+            if file.basename.find("CRT") > -1 and file.basename.find(ctx.attr.arch) > -1 and file.basename.find(msvc_version) > -1:
+                candle_in.append(file)
+                candle_arguments.append("-dMergeModulesBasePath=" + file.dirname)
+                candle_arguments.append("-dMergeModuleFileCRT=" + file.basename)
+                break
+
+    # pass in architecture
+    candle_arguments.append("-dPlatform=" + ctx.attr.arch)
+    candle_arguments.append("-arch")
+    candle_arguments.append(ctx.attr.arch)
+
+    # pass in extension files needed by wix
+    for extension in ctx.attr.extensions:
+        candle_arguments.append("-ext")
+        candle_arguments.append(extension)
+
+    # pass in .wxs files
+    output_directory = ""
+    for label in ctx.attr.srcs:
+        for file in label.files.to_list():
+            candle_in.append(file)
+            candle_arguments.append(file.path)
+
+            # wix output files are the input files with wixobj extension instead
+            output_file = ctx.actions.declare_file(file.basename.split(".")[0] + ".wixobj")
+            candle_out.append(output_file)
+            output_directory = output_file.dirname
+
+    candle_arguments.append("-dOutDir=" + output_directory)
+    candle_arguments.append("-dTargetDir=" + output_directory)
+    candle_arguments.append("-out")
+    candle_arguments.append(output_directory + "/")
+
+    python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
+    ctx.actions.run(
+        outputs = candle_out,
+        inputs = depset(transitive = [depset(candle_in), ctx.attr._candle_wrapper_script.files, python.files, ctx.attr._candle.files]),
+        executable = python.interpreter.path,
+        arguments = candle_arguments,
+    )
+
+    light_in = candle_out
+    light_out = []
+    light_arguments = ["-wx", "-cultures:null"]
+
+    # pass in extension files needed by wix
+    for extension in ctx.attr.extensions:
+        light_arguments.append("-ext")
+        light_arguments.append(extension)
+
+    # internal consistency evaluators to skip
+    for sice in ctx.attr.light_sice:
+        light_arguments.append("-sice:" + sice)
+
+    for file in candle_out:
+        light_arguments.append(file.path)
+
+    output_filename = ctx.label.name + "-" + mongo_version
+    light_msi = ctx.actions.declare_file(output_filename + ".pre.msi")
+    light_out.append(light_msi)
+    light_arguments.append("-out")
+    light_arguments.append(light_msi.path)
+
+    ctx.actions.run(
+        outputs = light_out,
+        inputs = light_in,
+        executable = ctx.executable._light,
+        arguments = light_arguments,
+    )
+
+    output_msi = ctx.actions.declare_file(output_filename + ".msi")
+    msi_trim_script = ctx.attr._msi_trim_script.files.to_list()[0].path
+
+    ctx.actions.run(
+        outputs = [output_msi],
+        inputs = depset(transitive = [depset(light_out), ctx.attr._msi_trim_script.files, python.files]),
+        executable = python.interpreter.path,
+        arguments = [msi_trim_script, light_msi.path, output_msi.path],
+    )
+
+    return [DefaultInfo(
+        files = depset([output_msi]),
+    )]
+
+mongo_msi = rule(
+    mongo_msi_impl,
+    attrs = {
+        "srcs": attr.label_list(allow_files = [".wxs"]),
+        "deps": attr.string_keyed_label_dict(allow_empty = True),
+        "custom_action": attr.label(allow_files = [".dll"]),
+        "extensions": attr.string_list(allow_empty = True),
+        "mongo_version": attr.string(mandatory = True),
+        "msvc_version": attr.string(mandatory = False),
+        "use_merge_modules": attr.bool(default = False),
+        "wix_vars": attr.string_dict(allow_empty = True),
+        "light_sice": attr.string_list(allow_empty = True),
+        "arch": attr.string(default = "x64"),
+        "_candle": attr.label(
+            default = "@wix_toolset//:candle",
+            allow_single_file = True,
+            executable = True,
+            cfg = "host",
+        ),
+        "_candle_wrapper_script": attr.label(
+            doc = "The python msi trimming script to use.",
+            default = "//buildscripts:candle_wrapper.py",
+            allow_single_file = True,
+        ),
+        "_light": attr.label(
+            default = "@wix_toolset//:light",
+            allow_single_file = True,
+            executable = True,
+            cfg = "host",
+        ),
+        "_msi_trim_script": attr.label(
+            doc = "The python msi trimming script to use.",
+            default = "//buildscripts:msitrim.py",
+            allow_single_file = True,
+        ),
+        "_merge_modules": attr.label(
+            default = "@local_windows_msvc//:merge_modules",
+        ),
+    },
+    doc = "Create a msi using wix toolset",
+    toolchains = ["@bazel_tools//tools/python:toolchain_type"],
+)

bazel/install_rules/windows_msvc.bzl

@@ -0,0 +1,26 @@
+def find_windows_msvc(ctx):
+    vc_dir = ctx.os.environ.get("VCINSTALLDIR")
+    if vc_dir:
+        ctx.symlink(vc_dir + "Redist/MSVC", "msvc")
+        ctx.file(
+            "BUILD.bazel",
+            """
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "merge_modules",
+    srcs = select({
+        "@platforms//os:windows": glob(["**/*.msm"]),
+        "//conditions:default": [],
+    }),
+)
+""",
+        )
+    else:
+        fail("Environment variable VCINSTALLDIR must be set to find local msvc.")
+    return None
+
+windows_msvc = repository_rule(
+    implementation = find_windows_msvc,
+    environ = ["VCINSTALLDIR"],
+)

buildscripts/BUILD.bazel

@@ -1,8 +1,10 @@
 load("@poetry//:dependencies.bzl", "dependency")
 
 exports_files([
+    "candle_wrapper.py",
     "cheetah_source_generator.py",
     "clang_tidy_config_gen.py",
+    "msitrim.py",
 ])
 
 py_binary(

buildscripts/candle_wrapper.py

@@ -0,0 +1,36 @@
+import hashlib
+import subprocess
+import sys
+import uuid
+
+# Wrapper script to substitute GENERATE_UPGRADE_CODE as starlark can't create hashes
+candle_args = sys.argv[1:]
+generate_code_replace_text = "GENERATE_UPGRADE_CODE"
+
+# If our args contain GENERATE_UPGRADE_CODE we need to generate it, otherwise we can just call candle
+if any(generate_code_replace_text in x for x in candle_args):
+    # We must regenerate upgrade codes for each major release. These upgrade codes must also be
+    # different for each MSI edition. To generate these we must be passed MongoDBMajorVersion and Edition
+    # These are are things like -dMongoDBMajorVersion=8.1 and -dEdition=SSL
+    for i, arg in enumerate(candle_args):
+        if generate_code_replace_text in arg:
+            upgrade_code_index = i
+        if "MongoDBMajorVersion" in arg:
+            major_version = arg.split("=")[1]
+        if "Edition" in arg:
+            msi_edition = arg.split("=")[1]
+
+    # Create uuid for upgrade code
+    m = hashlib.sha256()
+    hash_str = "{}_{}".format(major_version, msi_edition)
+    m.update(hash_str.encode())
+    upgrade_code = str(uuid.UUID(bytes=m.digest()[0:16]))
+
+    candle_args[upgrade_code_index] = candle_args[upgrade_code_index].replace(
+        generate_code_replace_text, upgrade_code
+    )
+
+    # run candle with updated args
+    subprocess.call(candle_args)
+else:
+    subprocess.call(candle_args)

buildscripts/msitrim.py

@@ -29,13 +29,14 @@ def exec_update(db, query, column, value):
 def main():
     """Execute Main program."""
     parser = argparse.ArgumentParser(description="Trim MSI.")
-    parser.add_argument("file", type=argparse.FileType("r"), help="file to trim")
-    parser.add_argument("out", type=argparse.FileType("w"), help="file to output to")
+    parser.add_argument("file", type=str, help="file to trim")
+    parser.add_argument("out", type=str, help="file to output to")
 
     args = parser.parse_args()
     print("Trimming MSI")
 
-    db = msilib.OpenDatabase(args.file.name, msilib.MSIDBOPEN_DIRECT)
+    shutil.copyfile(args.file, args.out)
+    db = msilib.OpenDatabase(args.out, msilib.MSIDBOPEN_DIRECT)
 
     exec_delete(
         db,
@@ -56,8 +57,6 @@ def main():
 
     db.Commit()
 
-    shutil.copyfile(args.file.name, args.out.name)
-
 
 if __name__ == "__main__":
     main()

buildscripts/packaging/msi/BUILD.bazel

@@ -0,0 +1,9 @@
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "msi_project_files",
+    srcs = select({
+        "@platforms//os:windows": glob(["*"]),
+        "//conditions:default": [],
+    }),
+)

distsrc/BUILD.bazel

@@ -0,0 +1,6 @@
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "license_files",
+    srcs = glob(["*"]),
+)

src/mongo/installer/compass/BUILD.bazel

@@ -0,0 +1,9 @@
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "compass_files",
+    srcs = select({
+        "@platforms//os:windows": ["Install-Compass.ps1"],
+        "//conditions:default": ["install_compass"],
+    }),
+)

src/mongo/installer/msi/BUILD.bazel

@@ -0,0 +1,76 @@
+load("//bazel/install_rules:msi.bzl", "mongo_msi")
+
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "install_bins",
+    srcs = [
+        "//:install-compass",
+        "//:install-dist-test",
+    ],
+)
+
+mongo_msi(
+    name = "mongodb-win32-x86_64-windows",
+    srcs = [
+        "wxs/BinaryFragment.wxs",
+        "wxs/FeatureFragment.wxs",
+        "wxs/Installer_64.wxs",
+        "wxs/LicensingFragment.wxs",
+        "wxs/UIFragment.wxs",
+    ],
+    custom_action = "//src/mongo/installer/msi/ca:mongoca",
+    exec_properties = {
+        "no-cache": "1",
+        "no-remote": "1",
+        "local": "1",
+    },
+    extensions = [
+        "WixUtilExtension.dll",
+        "WixUIExtension.dll",
+    ],
+    # Suppress VC140_CRT_CRT.MSM Internal Consistency Errors
+    # ICE82 - Suppress "duplicate sequence number"
+    # -- https://msdn.microsoft.com/en-us/library/windows/desktop/aa368798(v=vs.85).aspx
+
+    # ICE03 - Supress "String overflow"
+    # -- https://msdn.microsoft.com/en-us/library/windows/desktop/aa369037(v=vs.85).aspx
+
+    # ICE30 - Suppress "different components install same file"
+    # -- mongod.exe is installed in two different components but only one is ever used during an install
+    #    so this consistency check can be ignored.
+    # -- https://msdn.microsoft.com/en-us/library/windows/desktop/aa368954(v=vs.85).aspx
+    light_sice = [
+        "ICE82",
+        "ICE03",
+        "ICE30",
+    ],
+    mongo_version = "MONGO_VERSION",
+    msvc_version = "MSVC_VERSION",
+    target_compatible_with = select({
+        "@platforms//os:windows": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    use_merge_modules = True,
+    wix_vars = {
+        "ProductId": "*",
+        "UpgradeCode": "GENERATE_UPGRADE_CODE",
+        "Configuration": "Release",
+        "Flavor": "2008R2Plus",
+        "ProjectName": "MongoDB",
+        "TargetExt": ".msi",
+    } | select({
+        "//bazel/config:build_enterprise_enabled": {"Edition": "Enterprise"},
+        "//bazel/config:ssl_enabled_build_enterprise_disabled": {"Edition": "SSL"},
+        "//conditions:default": {"Edition": "Standard"},
+    }),
+    deps = {
+        "BinarySource": ":install_bins",
+        "ProjectDir": "//buildscripts/packaging/msi:msi_project_files",
+        "SaslSource": "@windows_sasl//:bins",
+        "LicenseSource": "//distsrc:license_files",
+    } | select({
+        "//bazel/config:build_enterprise_enabled": {"EnterpriseLicenseSource": "//src/mongo/db/modules/enterprise/distsrc:enterprise_license_files"},
+        "//conditions:default": {},
+    }),
+)

src/mongo/installer/msi/SConscript

@@ -50,7 +50,7 @@ buildDir = env.Dir("$BUILD_DIR").path
 buildRoot = env.Dir("$BUILD_ROOT").path
 toolBuildDir = buildDir + r"\mongo"
 
-enterprisebase = "src\mongo\db\modules\enterprise"
+enterpriselicensesource = "src\mongo\db\modules\enterprise\distsrc"
 enterpriseToolBuildDir = buildDir + r"\mongo\db\modules\enterprise"
 
 # Set up parameters to pass to wix -
@@ -109,8 +109,9 @@ candle_targets = env.Command(
         + " -dMongoDBVersion="
         + full_version
         + " -dLicenseSource=distsrc"
-        r" -dEnterpriseBase=" + enterprisebase + "\\"
-        " -dBinarySource="
+        r" -dEnterpriseLicenseSource="
+        + enterpriselicensesource
+        + " -dBinarySource="
         + '"$DESTDIR\\$PREFIX_BINDIR"'
         + " -dMergeModulesBasePath="
         + "\"${MSVS['VCREDISTMERGEMODULEPATH']}\""

src/mongo/installer/msi/wxs/LicensingFragment.wxs

@@ -5,7 +5,7 @@
     <DirectoryRef Id="INSTALLLOCATION">
       <?if $(var.Edition) = Enterprise ?>
         <Component Id="c_License" Guid="C0EF85E2-95F8-468B-BA95-2F739C63D2D7">
-          <File Id="f_License" Name="LICENSE-Enterprise.txt" Source="$(var.EnterpriseBase)\distsrc\LICENSE-Enterprise.txt"
+          <File Id="f_License" Name="LICENSE-Enterprise.txt" Source="$(var.EnterpriseLicenseSource)\LICENSE-Enterprise.txt"
                 DiskId ="1" KeyPath="yes" />
         </Component>
         <Component Id="c_Thirdparty" Guid="C45662A7-5C18-4A53-8D14-7C92DB38F13E">
@@ -13,7 +13,7 @@
                 DiskId ="1" KeyPath="yes" />
         </Component>
         <Component Id="c_ThirdpartyEnterprise" Guid="4D53FF2F-DB47-41F3-8C5C-4C397736724E">
-          <File Id="f_ThirdpartyEnterprise" Name="THIRD-PARTY-NOTICES.windows" Source="$(var.EnterpriseBase)\distsrc\THIRD-PARTY-NOTICES.windows"
+          <File Id="f_ThirdpartyEnterprise" Name="THIRD-PARTY-NOTICES.windows" Source="$(var.EnterpriseLicenseSource)\THIRD-PARTY-NOTICES.windows"
                 DiskId ="1" KeyPath="yes" />
         </Component>
 
@@ -56,7 +56,7 @@
     </ComponentGroup>
 
     <?if $(var.Edition) = Enterprise ?>
-    <WixVariable Id="WixUILicenseRtf" Value="$(var.EnterpriseBase)\distsrc\LICENSE-Enterprise.rtf" />
+    <WixVariable Id="WixUILicenseRtf" Value="$(var.EnterpriseLicenseSource)\LICENSE-Enterprise.rtf" />
     <?else ?>
     <WixVariable Id="WixUILicenseRtf" Value="$(var.ProjectDir)LICENSE-Community.rtf" />
     <?endif ?>