diff --git a/buildscripts/debugsymb_mapper.py b/buildscripts/debugsymb_mapper.py index d85ad64d9c9..f09418cac5f 100644 --- a/buildscripts/debugsymb_mapper.py +++ b/buildscripts/debugsymb_mapper.py @@ -157,19 +157,20 @@ class Mapper: default_web_service_base_url: str = "https://symbolizer-service.server-tig.prod.corp.mongodb.com" default_cache_dir = os.path.join(os.getcwd(), 'build', 'symbols_cache') - selected_binaries = ('mongos.debug', 'mongod.debug', 'mongo.debug') + selected_binaries = ('mongos', 'mongod', 'mongo') default_client_credentials_scope = "servertig-symbolizer-fullaccess" default_client_credentials_user_name = "client-user" default_creds_file_path = os.path.join(os.getcwd(), '.symbolizer_credentials.json') - def __init__(self, evg_version: str, evg_variant: str, client_id: str, client_secret: str, - cache_dir: str = None, web_service_base_url: str = None, + def __init__(self, evg_version: str, evg_variant: str, is_san_variant: bool, client_id: str, + client_secret: str, cache_dir: str = None, web_service_base_url: str = None, logger: logging.Logger = None): """ Initialize instance. :param evg_version: Evergreen version ID. :param evg_variant: Evergreen build variant name. + :param is_san_variant: Whether build variant is sanitizer build. :param client_id: Client id for Okta Oauth. :param client_secret: Secret key for Okta Oauth. :param cache_dir: Full path to cache directory as a string. @@ -178,6 +179,7 @@ class Mapper: """ self.evg_version = evg_version self.evg_variant = evg_variant + self.is_san_variant = is_san_variant self.cache_dir = cache_dir or self.default_cache_dir self.web_service_base_url = web_service_base_url or self.default_web_service_base_url @@ -263,11 +265,13 @@ class Mapper: urlinfo = self.multiversion_setup.get_urls(self.evg_version, self.evg_variant) - download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz", None) binaries_url = urlinfo.urls.get("Binaries", "") - - if not download_symbols_url: - download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.zip", None) + if self.is_san_variant: + # Sanitizer builds are not stripped and contain debug symbols + download_symbols_url = binaries_url + else: + download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz") or urlinfo.urls.get( + "mongo-debugsymbols.zip") if not download_symbols_url: self.logger.error("Couldn't find URL for debug symbols. Version: %s, URLs dict: %s", @@ -319,23 +323,17 @@ class Mapper: extractor = CmdOutputExtractor() - debug_symbols_path = self.download(self.debug_symbols_url) - debug_symbols_unpacked_path = self.unpack(debug_symbols_path) - binaries_path = self.download(self.url) binaries_unpacked_path = self.unpack(binaries_path) - # we need to analyze two directories: main binary folder inside debug-symbols and + # we need to analyze two directories: main binary folder and # shared libraries folder inside binaries. # main binary folder holds main binaries, like mongos, mongod, mongo ... # shared libraries folder holds shared libraries, tons of them. # some build variants do not contain shared libraries. - debug_symbols_unpacked_path = os.path.join(debug_symbols_unpacked_path, 'dist-test') binaries_unpacked_path = os.path.join(binaries_unpacked_path, 'dist-test') - self.logger.info("INSIDE unpacked debug-symbols/dist-test: %s", - os.listdir(debug_symbols_unpacked_path)) self.logger.info("INSIDE unpacked binaries/dist-test: %s", os.listdir(binaries_unpacked_path)) @@ -352,19 +350,19 @@ class Mapper: # start with main binary folder for binary in self.selected_binaries: - full_bin_path = os.path.join(debug_symbols_unpacked_path, + full_bin_path = os.path.join(binaries_unpacked_path, self.path_options.main_binary_folder_name, binary) if not os.path.exists(full_bin_path): self.logger.error("Could not find binary at %s", full_bin_path) - return + continue build_id_output = extractor.get_build_id(full_bin_path) if not build_id_output.build_id: self.logger.error("Build ID couldn't be extracted. \nReadELF output %s", build_id_output.cmd_output) - return + continue else: self.logger.info("Extracted build ID: %s", build_id_output.build_id) @@ -397,14 +395,14 @@ class Mapper: if not os.path.exists(sofile_path): self.logger.error("Could not find binary at %s", sofile_path) - return + continue build_id_output = extractor.get_build_id(sofile_path) if not build_id_output.build_id: self.logger.error("Build ID couldn't be extracted. \nReadELF out %s", build_id_output.cmd_output) - return + continue else: self.logger.info("Extracted build ID: %s", build_id_output.build_id) @@ -426,6 +424,7 @@ class Mapper: # mappings is a generator, we iterate over to generate mappings on the go for mapping in mappings: + self.logger.info("Creating mapping %s", mapping) response = self.http_client.post('/'.join((self.web_service_base_url, 'add')), json=mapping) if response.status_code != 200: @@ -440,11 +439,12 @@ def make_argument_parser(parser=None, **kwargs): if parser is None: parser = argparse.ArgumentParser(**kwargs) - parser.add_argument('--version') - parser.add_argument('--client-id') - parser.add_argument('--client-secret') - parser.add_argument('--variant') - parser.add_argument('--web-service-base-url', default="") + parser.add_argument("--version") + parser.add_argument("--client-id") + parser.add_argument("--client-secret") + parser.add_argument("--variant") + parser.add_argument("--is-san-variant", action="store_true") + parser.add_argument("--web-service-base-url", default="") return parser @@ -452,7 +452,8 @@ def main(options): """Execute mapper here. Main entry point.""" mapper = Mapper(evg_version=options.version, evg_variant=options.variant, - client_id=options.client_id, client_secret=options.client_secret, + is_san_variant=options.is_san_variant, client_id=options.client_id, + client_secret=options.client_secret, web_service_base_url=options.web_service_base_url) # when used as a context manager, mapper instance automatically cleans files/folders after finishing its job. diff --git a/evergreen/generate_buildid_debug_symbols_mapping.sh b/evergreen/generate_buildid_debug_symbols_mapping.sh index d1866e961a5..142614ce821 100644 --- a/evergreen/generate_buildid_debug_symbols_mapping.sh +++ b/evergreen/generate_buildid_debug_symbols_mapping.sh @@ -6,10 +6,16 @@ cd src set -o errexit set -o verbose +is_san_variant_arg="" +if [[ -n "${san_options}" ]]; then + is_san_variant_arg="--is-san-variant" +fi + activate_venv $python buildscripts/debugsymb_mapper.py \ --version "${version_id}" \ --client-id "${symbolizer_client_id}" \ --client-secret "${symbolizer_client_secret}" \ - --variant "${build_variant}" + --variant "${build_variant}" \ + $is_san_variant_arg