SERVER-76457 Support storing Build ID to debug symbols mapping for sanitizer builds

buildscripts/debugsymb_mapper.py

@@ -157,19 +157,20 @@ class Mapper:
 
     default_web_service_base_url: str = "https://symbolizer-service.server-tig.prod.corp.mongodb.com"
     default_cache_dir = os.path.join(os.getcwd(), 'build', 'symbols_cache')
-    selected_binaries = ('mongos.debug', 'mongod.debug', 'mongo.debug')
+    selected_binaries = ('mongos', 'mongod', 'mongo')
     default_client_credentials_scope = "servertig-symbolizer-fullaccess"
     default_client_credentials_user_name = "client-user"
     default_creds_file_path = os.path.join(os.getcwd(), '.symbolizer_credentials.json')
 
-    def __init__(self, evg_version: str, evg_variant: str, client_id: str, client_secret: str,
+    def __init__(self, evg_version: str, evg_variant: str, is_san_variant: bool, client_id: str,
-                 cache_dir: str = None, web_service_base_url: str = None,
+                 client_secret: str, cache_dir: str = None, web_service_base_url: str = None,
                  logger: logging.Logger = None):
         """
         Initialize instance.
 
         :param evg_version: Evergreen version ID.
         :param evg_variant: Evergreen build variant name.
+        :param is_san_variant: Whether build variant is sanitizer build.
         :param client_id: Client id for Okta Oauth.
         :param client_secret: Secret key for Okta Oauth.
         :param cache_dir: Full path to cache directory as a string.
@@ -178,6 +179,7 @@ class Mapper:
         """
         self.evg_version = evg_version
         self.evg_variant = evg_variant
+        self.is_san_variant = is_san_variant
         self.cache_dir = cache_dir or self.default_cache_dir
         self.web_service_base_url = web_service_base_url or self.default_web_service_base_url
 
@@ -263,11 +265,13 @@ class Mapper:
 
         urlinfo = self.multiversion_setup.get_urls(self.evg_version, self.evg_variant)
 
-        download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz", None)
         binaries_url = urlinfo.urls.get("Binaries", "")
-
+        if self.is_san_variant:
-        if not download_symbols_url:
+            # Sanitizer builds are not stripped and contain debug symbols
-            download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.zip", None)
+            download_symbols_url = binaries_url
+        else:
+            download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz") or urlinfo.urls.get(
+                "mongo-debugsymbols.zip")
 
         if not download_symbols_url:
             self.logger.error("Couldn't find URL for debug symbols. Version: %s, URLs dict: %s",
@@ -319,23 +323,17 @@ class Mapper:
 
         extractor = CmdOutputExtractor()
 
-        debug_symbols_path = self.download(self.debug_symbols_url)
-        debug_symbols_unpacked_path = self.unpack(debug_symbols_path)
-
         binaries_path = self.download(self.url)
         binaries_unpacked_path = self.unpack(binaries_path)
 
-        # we need to analyze two directories: main binary folder inside debug-symbols and
+        # we need to analyze two directories: main binary folder and
         # shared libraries folder inside binaries.
         # main binary folder holds main binaries, like mongos, mongod, mongo ...
         # shared libraries folder holds shared libraries, tons of them.
         # some build variants do not contain shared libraries.
 
-        debug_symbols_unpacked_path = os.path.join(debug_symbols_unpacked_path, 'dist-test')
         binaries_unpacked_path = os.path.join(binaries_unpacked_path, 'dist-test')
 
-        self.logger.info("INSIDE unpacked debug-symbols/dist-test: %s",
-                         os.listdir(debug_symbols_unpacked_path))
         self.logger.info("INSIDE unpacked binaries/dist-test: %s",
                          os.listdir(binaries_unpacked_path))
 
@@ -352,19 +350,19 @@ class Mapper:
 
         # start with main binary folder
         for binary in self.selected_binaries:
-            full_bin_path = os.path.join(debug_symbols_unpacked_path,
+            full_bin_path = os.path.join(binaries_unpacked_path,
                                          self.path_options.main_binary_folder_name, binary)
 
             if not os.path.exists(full_bin_path):
                 self.logger.error("Could not find binary at %s", full_bin_path)
-                return
+                continue
 
             build_id_output = extractor.get_build_id(full_bin_path)
 
             if not build_id_output.build_id:
                 self.logger.error("Build ID couldn't be extracted. \nReadELF output %s",
                                   build_id_output.cmd_output)
-                return
+                continue
             else:
                 self.logger.info("Extracted build ID: %s", build_id_output.build_id)
 
@@ -397,14 +395,14 @@ class Mapper:
 
             if not os.path.exists(sofile_path):
                 self.logger.error("Could not find binary at %s", sofile_path)
-                return
+                continue
 
             build_id_output = extractor.get_build_id(sofile_path)
 
             if not build_id_output.build_id:
                 self.logger.error("Build ID couldn't be extracted. \nReadELF out %s",
                                   build_id_output.cmd_output)
-                return
+                continue
             else:
                 self.logger.info("Extracted build ID: %s", build_id_output.build_id)
 
@@ -426,6 +424,7 @@ class Mapper:
 
         # mappings is a generator, we iterate over to generate mappings on the go
         for mapping in mappings:
+            self.logger.info("Creating mapping %s", mapping)
             response = self.http_client.post('/'.join((self.web_service_base_url, 'add')),
                                              json=mapping)
             if response.status_code != 200:
@@ -440,11 +439,12 @@ def make_argument_parser(parser=None, **kwargs):
     if parser is None:
         parser = argparse.ArgumentParser(**kwargs)
 
-    parser.add_argument('--version')
+    parser.add_argument("--version")
-    parser.add_argument('--client-id')
+    parser.add_argument("--client-id")
-    parser.add_argument('--client-secret')
+    parser.add_argument("--client-secret")
-    parser.add_argument('--variant')
+    parser.add_argument("--variant")
-    parser.add_argument('--web-service-base-url', default="")
+    parser.add_argument("--is-san-variant", action="store_true")
+    parser.add_argument("--web-service-base-url", default="")
     return parser
 
 
@@ -452,7 +452,8 @@ def main(options):
     """Execute mapper here. Main entry point."""
 
     mapper = Mapper(evg_version=options.version, evg_variant=options.variant,
-                    client_id=options.client_id, client_secret=options.client_secret,
+                    is_san_variant=options.is_san_variant, client_id=options.client_id,
+                    client_secret=options.client_secret,
                     web_service_base_url=options.web_service_base_url)
 
     # when used as a context manager, mapper instance automatically cleans files/folders after finishing its job.

evergreen/generate_buildid_debug_symbols_mapping.sh

@@ -6,10 +6,16 @@ cd src
 set -o errexit
 set -o verbose
 
+is_san_variant_arg=""
+if [[ -n "${san_options}" ]]; then
+  is_san_variant_arg="--is-san-variant"
+fi
+
 activate_venv
 
 $python buildscripts/debugsymb_mapper.py \
   --version "${version_id}" \
   --client-id "${symbolizer_client_id}" \
   --client-secret "${symbolizer_client_secret}" \
-  --variant "${build_variant}"
+  --variant "${build_variant}" \
+  $is_san_variant_arg