mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-80383 Add bazel support for compiling on Windows

This commit is contained in:
Zack Winter 2023-11-10 20:58:44 +00:00 committed by Evergreen Agent
parent 0a729cd810
commit 8165cf85a4
9 changed files with 179 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.bazelrc
View File

@ -1,13 +1,18 @@
# Reference docs: https://bazel.build/run/bazelrc
# This automatically sets a config based on host platform, ex: --config=macos if the host is macos
build --enable_platform_specific_config=true
# This makes builds more hermetic by preventing environment variables from leaking into the execution of rules
build --incompatible_strict_action_env=true
# Configure default values for platforms, host_platform, and crosstool_top.
# This supports the "crosstool" feature (aka building from our toolchain).
build --platforms=@mongo_toolchain//:platform
build --host_platform=@mongo_toolchain//:platform
build --crosstool_top=@mongo_toolchain//:toolchain_suite
# Currently the only platform with a custom toolchain config is linux, use the default
# toolchain_suite elsewhere.
build --platforms=@mongo_toolchain//:linux_arm64_gcc
build --host_platform=@mongo_toolchain//:linux_arm64_gcc
build:linux --crosstool_top=@mongo_toolchain//:toolchain_suite
# remote execution is the default, but only mongodb employees will be able to access
# the engflow cluster. External builders should use the local option below

7
bazel/config/BUILD.bazel
View File

@ -25,6 +25,13 @@ config_setting(
},
)
config_setting(
name = "compiler_type_msvc",
flag_values = {
"//bazel/config:compiler_type": "msvc",
},
)
# --------------------------------------
# Architecture + OS combinations
# --------------------------------------

16
bazel/mongo_src_rules.bzl
View File

@ -90,7 +90,8 @@ def mongo_cc_library(
tags: Tags to add to the rule.
copts: Any extra compiler options to pass in.
linkopts: Any extra link options to pass in.
linkstatic: Whether or not linkstatic should be passed to the native bazel cc_library rule.
linkstatic: Whether or not linkstatic should be passed to the native bazel cc_test rule. This argument
is ignored on windows since linking into DLLs is not currently supported.
local_defines: macro definitions passed to all source and header files.
"""
@ -109,7 +110,10 @@ def mongo_cc_library(
copts = MONGO_GLOBAL_COPTS + copts,
data = data,
tags = tags,
linkstatic = linkstatic,
linkstatic = select({
"@platforms//os:windows": True,
"//conditions:default": linkstatic,
}),
local_defines = MONGO_GLOBAL_DEFINES + local_defines,
includes = [],
)
@ -138,7 +142,8 @@ def mongo_cc_binary(
tags: Tags to add to the rule.
copts: Any extra compiler options to pass in.
linkopts: Any extra link options to pass in.
linkstatic: Whether or not linkstatic should be passed to the native bazel cc_test rule.
linkstatic: Whether or not linkstatic should be passed to the native bazel cc_test rule. This argument
is ignored on windows since linking into DLLs is not currently supported.
local_defines: macro definitions passed to all source and header files.
"""
@ -151,7 +156,10 @@ def mongo_cc_binary(
copts = MONGO_GLOBAL_COPTS + copts,
data = data,
tags = tags,
linkstatic = linkstatic,
linkstatic = select({
"@platforms//os:windows": True,
"//conditions:default": linkstatic,
}),
local_defines = MONGO_GLOBAL_DEFINES + LIBUNWIND_DEFINES + local_defines,
malloc = select({
"//bazel/config:tcmalloc_allocator": "//src/third_party/gperftools:tcmalloc_minimal",

37
bazel/toolchains/mongo_toolchain.BUILD
View File

@ -4,19 +4,30 @@ load("@//bazel/toolchains:mongo_cc_toolchain_config.bzl", "mongo_cc_toolchain_co
package(default_visibility = ["//visibility:public"])
# Establish a "platform" target so Bazel can pick the right platform for building:
[
platform(
name = "linux_arm64_" + compiler,
constraint_values = [
"@platforms//os:linux",
"@platforms//cpu:arm64",
"@bazel_tools//tools/cpp:" + compiler,
],
exec_properties = {
# debian gcc based image contains the base our toolchain needs (glibc version and build-essentials)
# https://hub.docker.com/layers/library/gcc/12.3-bookworm/images/sha256-6a3a5694d10299dbfb8747b98621abf4593bb54a5396999caa013cba0e17dd4f?context=explore
"container-image": "docker://docker.io/library/gcc@sha256:6a3a5694d10299dbfb8747b98621abf4593bb54a5396999caa013cba0e17dd4f",
}
)
for compiler in ["clang", "gcc"]
]
platform(
name = "platform",
name = "windows_amd64_msvc",
constraint_values = [
"@platforms//os:linux",
"@platforms//cpu:arm64",
"@bazel_tools//tools/cpp:gcc",
],
exec_properties = {
# debian gcc based image contains the base our toolchain needs (glibc version and build-essentials)
# https://hub.docker.com/layers/library/gcc/12.3-bookworm/images/sha256-6a3a5694d10299dbfb8747b98621abf4593bb54a5396999caa013cba0e17dd4f?context=explore
"container-image": "docker://docker.io/library/gcc@sha256:6a3a5694d10299dbfb8747b98621abf4593bb54a5396999caa013cba0e17dd4f",
}
"@platforms//cpu:x86_64",
"@platforms//os:windows",
"@bazel_tools//tools/cpp:msvc",
]
)
# Helper target for the toolchain (see below):
@ -156,6 +167,8 @@ cc_toolchain(
toolchain(
name = "mongo_toolchain",
toolchain = ":cc_mongo_toolchain",
toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
exec_compatible_with = [
"@platforms//os:linux",
"@platforms//cpu:arm64",
@ -165,8 +178,6 @@ toolchain(
"@platforms//os:linux",
"@platforms//cpu:arm64",
],
toolchain = ":cc_mongo_toolchain",
toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
)
cc_toolchain_suite(

56
etc/evergreen_yml_components/definitions.yml
View File

@ -2852,6 +2852,32 @@ tasks:
targets: src/mongo/db/commands:fsync_locked
compiler: clang
- name: compile_bazel_dist_test_windows
tags: []
depends_on:
- name: version_expansions_gen
variant: generate-tasks-for-version
commands:
# TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
- func: "fetch bazel"
- func: "scons compile"
vars:
targets: >-
build/fast/mongo/db/commands/fsync_locked.lib
# TODO SERVER-79852 remove "BAZEL_FLAGS=--config=local" flag
task_compile_flags: >-
BAZEL_BUILD_ENABLED=1
BAZEL_INTEGRATION_DEBUG=1
BAZEL_FLAGS=--config=local
ICECC=
--build-profile=fast
--ninja=disabled
--link-model=static
--modules=
- func: "verify build output present"
vars:
output: build/fast/mongo/db/commands/fsync_locked.lib
# Validates that the bazel mongo toolchain can be used to compile targets that
# contain C and asssembly source code files.
# TODO(SERVER-82195): simplify this to avoid duplication of parameters in other tests.
@ -2942,6 +2968,30 @@ tasks:
args: >-
--config=local
- name: run_bazel_program_windows
tags: []
depends_on:
- name: version_expansions_gen
variant: generate-tasks-for-version
commands:
# TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
- func: "fetch bazel"
- func: "bazel run"
vars:
target: >-
//src/mongo/platform:visibility_test1
# TODO SERVER-79852 remove "BAZEL_FLAGS=--config=local" flag
# TODO DEVPROD-2508 remove dbg flags when compilation_mode is synced with build_mode automatically
# TODO DEVPROD-2513 remove "--copt /std:c++20" flag when toolchain sets this automatically
args: >-
--config=local
--host_platform=@mongo_toolchain//:windows_amd64_msvc
--platforms=@mongo_toolchain//:windows_amd64_msvc
--//bazel/config:compiler_type=msvc
--compilation_mode=dbg
--//bazel/config:build_mode=dbg
--copt /std:c++20
## compile - build all scons targets except unittests ##
- name: compile_dist_test_half
tags: []
@ -9401,6 +9451,12 @@ task_groups:
- compile_bazel_program
- run_bazel_program
- <<: *compile_bazel_task_group_template
name: compile_bazel_windows_TG
tasks:
- compile_bazel_dist_test_windows
- run_bazel_program_windows
- <<: *compile_task_group_template
name: compile_upload_benchmarks_TG
tasks:

24
etc/evergreen_yml_components/variants/compile_static_analysis.yml
View File

@ -278,6 +278,30 @@ buildvariants:
tasks:
- name: compile_bazel_TG
# Note that this task is currently optional;
# This will eventually become suggested, then required.
- name: &windows-bazel-compile windows-bazel-compile
display_name: "Windows Bazel Compile"
cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
run_on:
- windows-vsCurrent-xlarge
stepback: false
expansions:
exe: ".exe"
ext: zip
content_type: application/zip
compile_flags: >-
--ssl
MONGO_DISTMOD=windows
CPPPATH="c:/sasl/include"
LIBPATH="c:/sasl/lib"
-j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.8")
--win-version-min=win10
python: '/cygdrive/c/python/python39/python.exe'
compile_variant: *windows-bazel-compile
tasks:
- name: compile_bazel_windows_TG
- <<: *linux-arm64-dynamic-compile-params
name: &amazon-linux2-arm64-dynamic-compile amazon-linux2-arm64-dynamic-compile
display_name: "! Amazon Linux 2 arm64 Shared Library Compile & Static Analysis"

24
evergreen/get_bazelisk.sh
View File

@ -1,7 +1,29 @@
#!/bin/bash
cd src
set -o errexit
set -o verbose
EXT=""
if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" || "$OSTYPE" == "win64" ]]; then
OS="windows"
EXT=".exe"
else
OS="linux"
fi
ARCH=$(uname -m)
if [[ "$ARCH" == "arm64" || "$ARCH" == "aarch64" ]]; then
ARCH="arm64"
else
ARCH="amd64"
fi
# TODO(SERVER-81038): remove once bazel/bazelisk is self-hosted.
curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-arm64 --output ./bazelisk && chmod +x ./bazelisk
CURL_COMMAND="curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-${OS}-${ARCH}${EXT} --output ./bazelisk"
echo $CURL_COMMAND
eval $CURL_COMMAND
chmod +x "./bazelisk"

38
site_scons/site_tools/integrate_bazel.py
View File

@ -11,11 +11,19 @@ import threading
import time
from typing import List, Dict, Set, Tuple
import urllib.request
import sys
import SCons
import mongo.platform as mongo_platform
import mongo.generators as mongo_generators
_SUPPORTED_PLATFORM_MATRIX = [
"windows:amd64:msvc",
"linux:arm64:gcc",
"linux:arm64:clang",
]
class Globals:
@ -58,8 +66,8 @@ def convert_scons_node_to_bazel_target(scons_node: SCons.Node.FS.File) -> str:
# convert to the source path i.e.: src/mongo/db/libcommands.so
bazel_path = scons_node.srcnode().path
# bazel uses source paths in the output i.e.: src/mongo/db
bazel_dir = os.path.dirname(bazel_path)
# bazel uses source paths in the output i.e.: src/mongo/db, replace backslashes on windows
bazel_dir = os.path.dirname(bazel_path).replace("\\", "/")
# extract the platform prefix for a given file so we can remove it i.e.: libcommands.so -> 'lib'
prefix = env.subst(scons_node.get_builder().get_prefix(env), target=[scons_node],
@ -298,32 +306,27 @@ def generate(env: SCons.Environment.Environment) -> None:
# === Architecture/platform ===
# Bail if current architecture not supported for Bazel:
current_architecture = platform.machine()
supported_architectures = ['aarch64']
if current_architecture not in supported_architectures:
normalized_arch = platform.machine().lower().replace("aarch64", "arm64")
normalized_os = sys.platform.replace("win32", "windows")
current_platform = f"{normalized_os}:{normalized_arch}:{env.ToolchainName()}"
if current_platform not in _SUPPORTED_PLATFORM_MATRIX:
raise Exception(
f'Bazel not supported on this architecture ({current_architecture}); supported architectures are: [{supported_architectures}]'
)
if not env.ToolchainIs('gcc', 'clang'):
raise Exception(
f"Unsupported Bazel c++ toolchain: {env.ToolchainName()} is neither `gcc` nor `clang`"
f'Bazel not supported on this platform ({current_platform}); supported platforms are: [{", ".join(_SUPPORTED_PLATFORM_MATRIX)}]'
)
# === Bazelisk ===
# TODO(SERVER-81038): remove once bazel/bazelisk is self-hosted.
if not os.path.exists("bazelisk"):
ext = ".exe" if normalized_os == "windows" else ""
urllib.request.urlretrieve(
"https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-arm64",
f"https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-{normalized_os}-{normalized_arch}{ext}",
"bazelisk")
os.chmod("bazelisk", stat.S_IXUSR)
# === Build settings ===
static_link = env.GetOption("link-model") in ["auto", "static"]
if not env.ToolchainIs('gcc', 'clang'):
raise Exception(f"Toolchain {env.ToolchainName()} is neither `gcc` nor `clang`")
if env.GetOption("release") is not None:
build_mode = "release"
@ -346,8 +349,10 @@ def generate(env: SCons.Environment.Environment) -> None:
f'--//bazel/config:use_lldbserver={False if env.GetOption("lldb-server") is None else True}',
f'--//bazel/config:use_wait_for_debugger={False if env.GetOption("wait-for-debugger") is None else True}',
f'--//bazel/config:use_disable_ref_track={False if env.GetOption("disable-ref-track") is None else True}',
f'--dynamic_mode={"off" if static_link else "fully"}',
f'--platforms=@mongo_toolchain//:{normalized_os}_{normalized_arch}_{env.ToolchainName()}',
f'--host_platform=@mongo_toolchain//:{normalized_os}_{normalized_arch}_{env.ToolchainName()}',
'--compilation_mode=dbg', # always build this compilation mode as we always build with -g
'--dynamic_mode=%s' % ('off' if static_link else 'fully'),
]
Globals.bazel_base_build_command = [
@ -360,7 +365,8 @@ def generate(env: SCons.Environment.Environment) -> None:
env['BAZEL_FLAGS_STR'] = str(bazel_internal_flags) + env.get("BAZEL_FLAGS", "")
# We always use --compilation_mode debug for now as we always want -g, so assume -dbg location
env["BAZEL_OUT_DIR"] = env.Dir("#/bazel-out/$TARGET_ARCH-dbg/bin/")
out_dir_platform = "x64_windows" if normalized_os == "windows" else "$TARGET_ARCH"
env["BAZEL_OUT_DIR"] = env.Dir(f"#/bazel-out/{out_dir_platform}-dbg/bin/")
# === Builders ===
create_library_builder(env)

6
src/third_party/gperftools/BUILD.bazel vendored
View File

@ -45,7 +45,6 @@ mongo_cc_library(
"dist/src/thread_cache.cc",
] + select({
"@platforms//os:windows": [
"dist/src/tcmalloc.cc",
"dist/src/windows/port.cc",
"dist/src/windows/system-alloc.cc",
"dist/src/fake_stacktrace_scope.cc",
@ -56,8 +55,9 @@ mongo_cc_library(
"dist/src/system-alloc.cc",
],
}) + select({
"@//bazel/config:dbg": ["dist/src/debugallocation.cc"],
"//conditions:default": ["dist/src/tcmalloc.cc"],
"@//bazel/config:linux_dbg": ["dist/src/debugallocation.cc"],
"@platforms//os:linux": ["dist/src/tcmalloc.cc"],
"//conditions:default": [],
}),
copts = ["-Isrc/third_party/gperftools/dist/src"] + select({
"@//bazel/config:linux_aarch64": [