SERVER-101282 implement pretty printer tests (#32918)

GitOrigin-RevId: f9e7badb8c161e607800577a94f6603d7327f6bd

BUILD.bazel

@@ -211,6 +211,11 @@ mongo_install(
         "//bazel/config:include_mongot_enabled": ["//:mongot_folder"],
         "//conditions:default": [],
     }),
+    pretty_printer_tests = {
+        "//src/mongo/util:pretty_printer_test.py": "//src/mongo/util:pretty_printer_test_program",
+        "//src/mongo/db/concurrency:lock_gdb_test.py": "//src/mongo/db:mongod",
+        "//src/mongo/db/query/optimizer:optimizer_gdb_test.py": "//src/mongo/db/query/optimizer:optimizer_gdb_test_program",
+    },
     deps = [
         "//src/mongo/db/modules/enterprise:dist-test",
     ],

bazel/install_rules/BUILD.bazel

@@ -3,4 +3,5 @@ package(default_visibility = ["//visibility:public"])
 # Expose script for external usage through bazel.
 exports_files([
     "install_rules.py",
+    "pretty_printer_test_creator.py",
 ])

bazel/install_rules/install_rules.bzl

@@ -7,6 +7,8 @@ load("@rules_pkg//pkg:providers.bzl", "PackageFilesInfo")
 load("@bazel_skylib//lib:paths.bzl", "paths")
 load("//bazel:mongo_src_rules.bzl", "SANITIZER_DATA", "SANITIZER_ENV")
 load("//bazel:separate_debug.bzl", "TagInfo")
+load("//bazel/install_rules:pretty_printer_tests.bzl", "mongo_pretty_printer_test")
+load("//bazel/install_rules:providers.bzl", "TestBinaryInfo")
 
 MongoInstallInfo = provider(
     doc = "A install rule provider to pass around deps files",
@@ -40,13 +42,6 @@ TEST_TAGS = {
     "storage_bm": 1,
 }
 
-TestBinaryInfo = provider(
-    doc = "Test binaries returned by this target.",
-    fields = {
-        "test_binaries": "group of all test binaries",
-    },
-)
-
 def test_binary_aspect_impl(target, ctx):
     """Collect all test binaries from transitive srcs and deps
 
@@ -59,6 +54,9 @@ def test_binary_aspect_impl(target, ctx):
     """
     transitive_deps = []
 
+    if TestBinaryInfo in target:
+        return []
+
     if TagInfo in target:
         for tag in target[TagInfo].tags:
             if tag in TEST_TAGS:
@@ -171,7 +169,7 @@ def sort_file(ctx, file, install_dir, file_map, is_directory):
     bin_install = install_dir + "/bin/" + basename
     lib_install = install_dir + "/lib/" + basename
 
-    if is_binary_file(ctx, basename):
+    if is_binary_file(ctx, basename) or basename.endswith(".py"):
         if not is_debug_file(ctx, basename):
             if ctx.attr.debug != "debug":
                 file_map["binaries"][file] = declare_output(ctx, bin_install, is_directory)
@@ -245,7 +243,7 @@ def mongo_install_rule_impl(ctx):
     input_deps = []
     installed_tests = []
     for file in test_files:
-        if not is_debug_file(ctx, file.basename):
+        if not is_debug_file(ctx, file.basename) and ctx.attr.debug != "debug":
             installed_tests.append(file_map["binaries"][file.path].path)
 
     installed_test_list_file = None
@@ -256,6 +254,8 @@ def mongo_install_rule_impl(ctx):
             content = "\n".join(installed_tests),
         )
         input_deps.append(installed_test_list_file)
+        real_test_list_output_location = ctx.actions.declare_file(install_dir + "/" + installed_test_list_file.basename)
+        outputs.append(real_test_list_output_location)
 
     # create a dep file for passing all the files we intend to install
     # to the python script
@@ -363,6 +363,7 @@ def mongo_install(
         deps = [],
         target_compatible_with = [],
         testonly = False,
+        pretty_printer_tests = {},
         **kwargs):
     """Perform install actions
 
@@ -390,6 +391,7 @@ def mongo_install(
     # stripped: only install bins, only available with separate_debug=True
     # debug: only install debug, only available with separate_debug=True
     for install_type in ["", "-stripped", "-debug"]:
+        modified_srcs = srcs
         install_target = "install-" + name + install_type
         debug = ""
         if install_type:
@@ -421,9 +423,21 @@ def mongo_install(
         if install_type:
             seperate_debug_incompat = ["@platforms//:incompatible"]
 
+        if len(pretty_printer_tests) > 0 and install_type != "-debug":
+            for test_script, test_binary in pretty_printer_tests.items():
+                pretty_printer_name = install_target + "-" + test_script.split(":")[-1].split(".")[0]
+                mongo_pretty_printer_test(
+                    name = pretty_printer_name,
+                    test_script = test_script,
+                    test_binary = test_binary,
+                    testonly = True,
+                )
+                modified_srcs = modified_srcs + [pretty_printer_name]
+            testonly = True
+
         mongo_install_rule(
             name = install_target,
-            srcs = srcs,
+            srcs = modified_srcs,
             debug = debug,
             deps = select({
                 "//bazel/config:build_enterprise_enabled": dep_targets,

bazel/install_rules/pretty_printer_test_creator.py

@@ -0,0 +1,76 @@
+import argparse
+import re
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--gdb-test-script")
+parser.add_argument("--pip-requirements-script")
+parser.add_argument("--pretty-printer-output")
+parser.add_argument("--pretty-printer-launcher-infile")
+parser.add_argument("--gdb-path")
+# Because we 'install' these files to a different location then where they are created
+# we want to pass the final location of these files rather than where they currently are
+parser.add_argument("--final-binary-path")
+parser.add_argument("--final-pretty-printer-path")
+parser.add_argument("--pretty-printer-launcher-output")
+
+extra_lines = [
+    "import os,subprocess,sys,traceback",
+    "cmd = 'python -c \"import os,sys;print(os.linesep.join(sys.path).strip())\"'",
+    "paths = subprocess.check_output(cmd,shell=True).decode('utf-8').split()",
+    "sys.path.extend(paths)",
+    "symbols_loaded = False",
+    "try:",
+    "    if gdb.objfiles()[0].lookup_global_symbol('main') is not None:",
+    "        symbols_loaded = True",
+    "except Exception:",
+    "    pass",
+    "if not symbols_loaded:",
+    r"    gdb.write('Could not find main symbol, debug info may not be loaded.\n')",
+    r"    gdb.write('TEST FAILED -- No Symbols.\\\n')",
+    "    gdb.execute('quit 1', to_string=True)",
+    "else:",
+    r"    gdb.write('Symbols loaded.\n')",
+    "gdb.execute('set confirm off')",
+    "gdb.execute('source .gdbinit')",
+    "try:",
+    "    verify_requirements(executable='python3')",
+    "except MissingRequirements as ex:",
+    "    print(ex)",
+    "    print('continuing testing anyways!')",
+    "except Exception as exc:",
+    "    print('ERROR: failed while verifying requirements.')",
+    "    traceback.print_exc()",
+    "    sys.exit(1)",
+]
+
+args = parser.parse_args()
+
+with open(args.pretty_printer_output, "w") as pretty_printer_output:
+    with open(args.pip_requirements_script) as pip_requirements_script:
+        pretty_printer_output.write(pip_requirements_script.read())
+        pretty_printer_output.write("\n")
+    pretty_printer_output.write("\n".join(extra_lines))
+    pretty_printer_output.write("\n")
+    with open(args.gdb_test_script) as test_script:
+        pretty_printer_output.write(test_script.read())
+
+# Verbose and test_args are hardcoded because there are no users of those overrides currently
+replacements = {
+    "@VERBOSE@": "True",
+    "@pretty_printer_test_py@": args.final_pretty_printer_path,
+    "@gdb_path@": args.gdb_path,
+    "@pretty_printer_test_program@": args.final_binary_path,
+    "@test_args@": '[""]',
+}
+
+escaped_replacements = dict((re.escape(k), v) for k, v in replacements.items())
+pattern = re.compile("|".join(escaped_replacements.keys()))
+
+with open(args.pretty_printer_launcher_output, "w") as pretty_printer_launcher_output:
+    with open(args.pretty_printer_launcher_infile) as pretty_printer_launcher_infile:
+        replaced_text = pattern.sub(
+            lambda match: escaped_replacements[re.escape(match.group(0))],
+            pretty_printer_launcher_infile.read(),
+        )
+        pretty_printer_launcher_output.write(replaced_text)

bazel/install_rules/pretty_printer_tests.bzl

@@ -0,0 +1,78 @@
+load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
+load("//bazel/install_rules:providers.bzl", "TestBinaryInfo")
+
+# This will not currently work under bazel test/run until we have a version of gdb to use in bazel
+def mongo_pretty_printer_test_impl(ctx):
+    split_name = ctx.label.name.split("-")
+
+    # eg optimizer_gdb_test
+    short_name = split_name[-1]
+
+    # eg /install-dist-test/bin/
+    final_output_directory = "/" + "-".join(split_name[:-1]) + "/bin/"
+
+    runnable_binary = None
+    for file in ctx.attr.test_binary.files.to_list():
+        if file.extension == "" or file.extension == "exe":
+            runnable_binary = file
+
+    pretty_printer_directory = "pretty-printer"
+    if "stripped" in split_name:
+        pretty_printer_directory += "-stripped"
+    pretty_printer_directory += "/"
+
+    script_output = ctx.actions.declare_file(pretty_printer_directory + short_name + ".py")
+    launcher_output = ctx.actions.declare_file(pretty_printer_directory + "pretty_printer_test_launcher_" + short_name + ".py")
+
+    python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
+
+    inputs = depset(transitive = [
+        ctx.attr._pretty_printer_creation_script.files,
+        ctx.attr.test_script.files,
+        ctx.attr._pip_requirements_script.files,
+        ctx.attr._pretty_printer_launcher_infile.files,
+        python.files,
+    ])
+    outputs = [launcher_output, script_output]
+
+    ctx.actions.run(
+        executable = python.interpreter.path,
+        outputs = outputs,
+        inputs = inputs,
+        arguments = [
+            ctx.file._pretty_printer_creation_script.path,
+            "--gdb-test-script=" + ctx.file.test_script.path,
+            "--pip-requirements-script=" + ctx.file._pip_requirements_script.path,
+            "--pretty-printer-output=" + script_output.path,
+            "--pretty-printer-launcher-infile=" + ctx.file._pretty_printer_launcher_infile.path,
+            # TODO have a way to get to gdb from inside bazel
+            "--gdb-path=" + "/opt/mongodbtoolchain/v4/bin/gdb",
+            # This is due to us being dependent on the final location of installed binaries - ideally we don't do this and run the tests
+            # in place and not from another directory
+            "--final-binary-path=" + ctx.bin_dir.path + final_output_directory + runnable_binary.basename,
+            "--final-pretty-printer-path=" + ctx.bin_dir.path + final_output_directory + script_output.basename,
+            "--pretty-printer-launcher-output=" + launcher_output.path,
+        ],
+        mnemonic = "MongoPrettyPrinterTestCreation",
+    )
+
+    default_provider = DefaultInfo(executable = launcher_output, files = depset(outputs))
+    test_binary_provider = TestBinaryInfo(test_binaries = depset([launcher_output]))
+    return [default_provider, test_binary_provider]
+
+mongo_pretty_printer_test = rule(
+    mongo_pretty_printer_test_impl,
+    attrs = {
+        "test_script": attr.label(allow_single_file = True),
+        "test_binary": attr.label(),
+        # TODO have a way to get to gdb from inside bazel
+        #"_gdb": attr.label(allow_single_file = True, default = "//:gdb"),
+        "_pretty_printer_creation_script": attr.label(allow_single_file = True, default = "//bazel/install_rules:pretty_printer_test_creator.py"),
+        "_pip_requirements_script": attr.label(allow_single_file = True, default = "//site_scons/mongo:pip_requirements.py"),
+        "_pretty_printer_launcher_infile": attr.label(allow_single_file = True, default = "//src/mongo/util:pretty_printer_test_launcher.py.in"),
+    },
+    doc = "Create pretty printer tests",
+    toolchains = ["@bazel_tools//tools/python:toolchain_type"],
+    executable = True,
+    test = True,
+)

bazel/install_rules/providers.bzl

@@ -0,0 +1,6 @@
+TestBinaryInfo = provider(
+    doc = "Test binaries returned by this target.",
+    fields = {
+        "test_binaries": "group of all test binaries",
+    },
+)

site_scons/mongo/BUILD.bazel

@@ -0,0 +1,3 @@
+package(default_visibility = ["//visibility:public"])
+
+exports_files(["pip_requirements.py"])

src/mongo/db/concurrency/BUILD.bazel

@@ -6,6 +6,7 @@ exports_files(
     glob([
         "*.h",
         "*.cpp",
+        "*.py",
     ]),
 )
 

src/mongo/db/query/optimizer/BUILD.bazel

@@ -6,6 +6,7 @@ exports_files(
     glob([
         "*.h",
         "*.cpp",
+        "*.py",
     ]),
 )
 

src/mongo/util/BUILD.bazel

@@ -5,7 +5,12 @@ load("//bazel/config:render_template.bzl", "render_template")
 package(default_visibility = ["//visibility:public"])
 
 exports_files(
-    glob(["*.h"]) + glob(["*.cpp"]),
+    glob([
+        "*.h",
+        "*.cpp",
+        "*.py",
+        "*.in",
+    ]),
 )
 
 # Headers designed to be used without source code. Split these out