SERVER-104108 Add simple_build_id and use linkopt -S on AL2023 (#35238)

GitOrigin-RevId: 2f65683f1441b5424de6137fef03b1461d0ae1fb

.bazelrc

@@ -119,6 +119,7 @@ common --flag_alias=tsan=//bazel/config:tsan
 common --flag_alias=ubsan=//bazel/config:ubsan
 common --flag_alias=dbg_level=//bazel/config:dbg_level
 common --flag_alias=mongo_toolchain_version=//bazel/config:mongo_toolchain_version
+common --flag_alias=simple_build_id=//bazel/config:simple_build_id
 
 #############################################################################################################################
 # BUILD 'PROFILES' - this is the area to set up configurations of flags to be used by developers.

MODULE.bazel

@@ -176,3 +176,7 @@ register_toolchains(
 setup_local_host_values = use_repo_rule("//bazel/platforms:local_host_values.bzl", "setup_local_host_values")
 
 setup_local_host_values(name = "local_host_values")
+
+setup_evergreen_variables = use_repo_rule("//bazel/repository_rules:evergreen_variables.bzl", "setup_evergreen_variables")
+
+setup_evergreen_variables(name = "evergreen_variables")

bazel/config/BUILD.bazel

@@ -35,6 +35,7 @@ load(
     "separate_debug",
     "server_js",
     "shared_archive",
+    "simple_build_id",
     "skip_archive",
     "spider_monkey_dbg",
     "ssl",
@@ -2647,6 +2648,22 @@ selects.config_setting_group(
     ],
 )
 
+# --------------------------------------
+# simple_build_id
+# --------------------------------------
+
+bool_flag(
+    name = "simple_build_id",
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "simple_build_id_enabled",
+    flag_values = {
+        "//bazel/config:simple_build_id": "True",
+    },
+)
+
 # --------------------------------------
 # running_through_bazelisk
 # --------------------------------------

bazel/config/configs.bzl

@@ -391,6 +391,20 @@ compress_debug_compile = rule(
     build_setting = config.bool(flag = True),
 )
 
+# =========
+# simple_build_id
+# =========
+
+simple_build_id_provider = provider(
+    doc = "Replace linker build-id with a simpler one based off output file name.",
+    fields = ["enabled"],
+)
+
+simple_build_id = rule(
+    implementation = lambda ctx: simple_build_id_provider(enabled = ctx.build_setting_value),
+    build_setting = config.bool(flag = True),
+)
+
 # =========
 # detect_odr_violations
 # =========

bazel/mongo_src_rules.bzl

@@ -32,6 +32,7 @@ load(
     "extract_debuginfo_test",
 )
 load("@local_host_values//:local_host_values_set.bzl", "NUM_CPUS")
+load("@evergreen_variables//:evergreen_variables.bzl", "UNSAFE_COMPILE_VARIANT", "UNSAFE_VERSION_ID")
 
 # These will throw an error if the following condition is not met:
 # (libunwind == on && os == linux) || libunwind == off || libunwind == auto
@@ -814,6 +815,9 @@ def _mongo_cc_binary_and_test(
             "//bazel/config:thin_lto_enabled": ["-Wl,--threads=" + str(NUM_CPUS)],
             "//bazel/config:bolt_enabled": ["-Wl,--threads=" + str(NUM_CPUS)],
             "//conditions:default": [],
+        }) + select({
+            "//bazel/config:simple_build_id_enabled": ["-Wl,--build-id=0x%x%x" % (hash(name), hash(str(UNSAFE_VERSION_ID) + str(UNSAFE_COMPILE_VARIANT)))],
+            "//conditions:default": [],
         }),
         "linkstatic": LINKSTATIC_ENABLED,
         "local_defines": MONGO_GLOBAL_DEFINES + local_defines,
@@ -822,6 +826,9 @@ def _mongo_cc_binary_and_test(
         "features": SKIP_ARCHIVE_FEATURE + ["-pic", "pie"] + features + select({
             "//bazel/config:windows_debug_symbols_enabled": ["generate_pdb_file"],
             "//conditions:default": [],
+        }) + select({
+            "//bazel/config:simple_build_id_enabled": ["-build_id"],
+            "//conditions:default": [],
         }),
         "dynamic_deps": select({
             "//bazel/config:linkstatic_disabled": deps,

bazel/repository_rules/evergreen_variables.bzl

@@ -0,0 +1,20 @@
+def _setup_evergreen_variables(ctx):
+    compile_variant = ctx.os.environ.get("compile_variant")
+    version_id = ctx.os.environ.get("version_id")
+
+    ctx.file(
+        "BUILD.bazel",
+        "",
+    )
+    ctx.file(
+        "evergreen_variables.bzl",
+        """
+UNSAFE_COMPILE_VARIANT = "%s"
+UNSAFE_VERSION_ID = "%s"
+""" % (compile_variant, version_id),
+    )
+
+setup_evergreen_variables = repository_rule(
+    implementation = _setup_evergreen_variables,
+    environ = ["compile_variant", "version_id"],
+)

bazel/toolchains/cc/mongo_linux/mongo_linux_cc_toolchain_config.bzl

@@ -1067,9 +1067,6 @@ def _impl(ctx):
             flag_set(
                 actions = all_link_actions,
                 flag_groups = [flag_group(flags = [
-                    # Explicitly enable GNU build id's if the linker supports it.
-                    "-Wl,--build-id",
-
                     # Explicitly use the new gnu hash section if the linker offers it.
                     "-Wl,--hash-style=gnu",
 
@@ -1088,6 +1085,20 @@ def _impl(ctx):
         ],
     )
 
+    build_id_feature = feature(
+        name = "build_id",
+        enabled = ctx.attr.compiler == COMPILERS.CLANG or ctx.attr.compiler == COMPILERS.GCC,
+        flag_sets = [
+            flag_set(
+                actions = all_link_actions,
+                flag_groups = [flag_group(flags = [
+                    # Explicitly enable GNU build id's if the linker supports it.
+                    "-Wl,--build-id",
+                ])],
+            ),
+        ],
+    )
+
     global_libs_feature = feature(
         name = "global_libs",
         enabled = True,
@@ -1210,6 +1221,7 @@ def _impl(ctx):
         compress_debug_sections_feature,
         rdynamic_feature,
         global_libs_feature,
+        build_id_feature,
     ]
 
     return [

etc/evergreen_yml_components/definitions.yml

@@ -2560,6 +2560,19 @@ functions:
       content_type: application/gzip
       display_name: Bazel JVM dump
 
+  "save bazel exec logs":
+    command: s3.put
+    params:
+      optional: true
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      local_file: src/bazel_exec.log
+      content_type: application/octet-stream
+      remote_file: ${project}/${build_variant}/${revision}/artifacts/bazel-exec.log.${build_id}-${task_name}.${execution}
+      bucket: mciuploads
+      permissions: public-read
+      display_name: Bazel exec log
+
   ### Attach report & artifacts ###
   "create bazel test report":
     command: subprocess.exec

etc/evergreen_yml_components/tasks/compile_tasks.yml

@@ -54,6 +54,7 @@ variables:
       - func: "save failed unittests"
       - func: "save bazel headers"
       - func: "save bazel jvm dump"
+      - func: "save bazel exec logs"
       - func: "save hang analyzer debugger files"
       - func: "save disk statistics"
       - func: "save system resource information"

etc/evergreen_yml_components/tasks/compile_tasks_shared.yml

@@ -28,6 +28,7 @@ variables:
       - func: "save failed unittests"
       - func: "save bazel headers"
       - func: "save bazel jvm dump"
+      - func: "save bazel exec logs"
       - func: "save hang analyzer debugger files"
       - func: "save disk statistics"
       - func: "save system resource information"

etc/evergreen_yml_components/variants/amazon/test_dev.yml

@@ -94,6 +94,7 @@ buildvariants:
       download_mongot_release: true
       compile_variant: *amazon-linux2023-arm64-static-compile
       evergreen_remote_exec: on
+      skip_debug_link: true
     tasks:
       - name: compile_test_parallel_unittest_stream_TG
       - name: compile_test_parallel_core_stream_and_pretty_printer_tests_TG

evergreen/bazel_compile.sh

@@ -85,6 +85,22 @@ if [ "${build_mongot}" = "true" ]; then
   bazel_args="${bazel_args} --include_mongot=True"
 fi
 
+# This is hacky way to pass off build time from archive_dist_test to archive_dist_test_debug
+# We create stripped binaries in archive_dist_test to avoid link time due to debug symbols
+# We then create the symbols normally in archive_dist_test_debug. We have to force the
+# build-id for debugging as they will be different when -Wl,-S is passed in.
+# The relinked binaries should still be hash identical when stripped with strip
+if [ "${skip_debug_link}" = "true" ]; then
+  export compile_variant="${compile_variant}"
+  export version_id="${version_id}"
+  if [ "${task_name}" = "archive_dist_test" ]; then
+    task_compile_flags="${task_compile_flags} --simple_build_id=True --linkopt='-Wl,-S' --separate_debug=False"
+  fi
+  if [ "${task_name}" = "archive_dist_test_debug" ]; then
+    task_compile_flags="${task_compile_flags} --simple_build_id=True"
+  fi
+fi
+
 set -o pipefail
 
 # Use `eval` to force evaluation of the environment variables in the echo statement: