diff --git a/BUILD.bazel b/BUILD.bazel index 1ad84451a96..f98cde7f9e6 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -211,6 +211,11 @@ mongo_install( "//bazel/config:include_mongot_enabled": ["//:mongot_folder"], "//conditions:default": [], }), + pretty_printer_tests = { + "//src/mongo/util:pretty_printer_test.py": "//src/mongo/util:pretty_printer_test_program", + "//src/mongo/db/concurrency:lock_gdb_test.py": "//src/mongo/db:mongod", + "//src/mongo/db/query/optimizer:optimizer_gdb_test.py": "//src/mongo/db/query/optimizer:optimizer_gdb_test_program", + }, deps = [ "//src/mongo/db/modules/enterprise:dist-test", ], diff --git a/bazel/install_rules/BUILD.bazel b/bazel/install_rules/BUILD.bazel index 183cd0b3d58..7b5a4c31130 100644 --- a/bazel/install_rules/BUILD.bazel +++ b/bazel/install_rules/BUILD.bazel @@ -3,4 +3,5 @@ package(default_visibility = ["//visibility:public"]) # Expose script for external usage through bazel. exports_files([ "install_rules.py", + "pretty_printer_test_creator.py", ]) diff --git a/bazel/install_rules/install_rules.bzl b/bazel/install_rules/install_rules.bzl index 08ae48e86e0..8b1f9f9f561 100644 --- a/bazel/install_rules/install_rules.bzl +++ b/bazel/install_rules/install_rules.bzl @@ -7,6 +7,8 @@ load("@rules_pkg//pkg:providers.bzl", "PackageFilesInfo") load("@bazel_skylib//lib:paths.bzl", "paths") load("//bazel:mongo_src_rules.bzl", "SANITIZER_DATA", "SANITIZER_ENV") load("//bazel:separate_debug.bzl", "TagInfo") +load("//bazel/install_rules:pretty_printer_tests.bzl", "mongo_pretty_printer_test") +load("//bazel/install_rules:providers.bzl", "TestBinaryInfo") MongoInstallInfo = provider( doc = "A install rule provider to pass around deps files", @@ -40,13 +42,6 @@ TEST_TAGS = { "storage_bm": 1, } -TestBinaryInfo = provider( - doc = "Test binaries returned by this target.", - fields = { - "test_binaries": "group of all test binaries", - }, -) - def test_binary_aspect_impl(target, ctx): """Collect all test binaries from transitive srcs and deps @@ -59,6 +54,9 @@ def test_binary_aspect_impl(target, ctx): """ transitive_deps = [] + if TestBinaryInfo in target: + return [] + if TagInfo in target: for tag in target[TagInfo].tags: if tag in TEST_TAGS: @@ -171,7 +169,7 @@ def sort_file(ctx, file, install_dir, file_map, is_directory): bin_install = install_dir + "/bin/" + basename lib_install = install_dir + "/lib/" + basename - if is_binary_file(ctx, basename): + if is_binary_file(ctx, basename) or basename.endswith(".py"): if not is_debug_file(ctx, basename): if ctx.attr.debug != "debug": file_map["binaries"][file] = declare_output(ctx, bin_install, is_directory) @@ -245,7 +243,7 @@ def mongo_install_rule_impl(ctx): input_deps = [] installed_tests = [] for file in test_files: - if not is_debug_file(ctx, file.basename): + if not is_debug_file(ctx, file.basename) and ctx.attr.debug != "debug": installed_tests.append(file_map["binaries"][file.path].path) installed_test_list_file = None @@ -256,6 +254,8 @@ def mongo_install_rule_impl(ctx): content = "\n".join(installed_tests), ) input_deps.append(installed_test_list_file) + real_test_list_output_location = ctx.actions.declare_file(install_dir + "/" + installed_test_list_file.basename) + outputs.append(real_test_list_output_location) # create a dep file for passing all the files we intend to install # to the python script @@ -363,6 +363,7 @@ def mongo_install( deps = [], target_compatible_with = [], testonly = False, + pretty_printer_tests = {}, **kwargs): """Perform install actions @@ -390,6 +391,7 @@ def mongo_install( # stripped: only install bins, only available with separate_debug=True # debug: only install debug, only available with separate_debug=True for install_type in ["", "-stripped", "-debug"]: + modified_srcs = srcs install_target = "install-" + name + install_type debug = "" if install_type: @@ -421,9 +423,21 @@ def mongo_install( if install_type: seperate_debug_incompat = ["@platforms//:incompatible"] + if len(pretty_printer_tests) > 0 and install_type != "-debug": + for test_script, test_binary in pretty_printer_tests.items(): + pretty_printer_name = install_target + "-" + test_script.split(":")[-1].split(".")[0] + mongo_pretty_printer_test( + name = pretty_printer_name, + test_script = test_script, + test_binary = test_binary, + testonly = True, + ) + modified_srcs = modified_srcs + [pretty_printer_name] + testonly = True + mongo_install_rule( name = install_target, - srcs = srcs, + srcs = modified_srcs, debug = debug, deps = select({ "//bazel/config:build_enterprise_enabled": dep_targets, diff --git a/bazel/install_rules/pretty_printer_test_creator.py b/bazel/install_rules/pretty_printer_test_creator.py new file mode 100644 index 00000000000..ee58fbda4e2 --- /dev/null +++ b/bazel/install_rules/pretty_printer_test_creator.py @@ -0,0 +1,76 @@ +import argparse +import re + +parser = argparse.ArgumentParser() + +parser.add_argument("--gdb-test-script") +parser.add_argument("--pip-requirements-script") +parser.add_argument("--pretty-printer-output") +parser.add_argument("--pretty-printer-launcher-infile") +parser.add_argument("--gdb-path") +# Because we 'install' these files to a different location then where they are created +# we want to pass the final location of these files rather than where they currently are +parser.add_argument("--final-binary-path") +parser.add_argument("--final-pretty-printer-path") +parser.add_argument("--pretty-printer-launcher-output") + +extra_lines = [ + "import os,subprocess,sys,traceback", + "cmd = 'python -c \"import os,sys;print(os.linesep.join(sys.path).strip())\"'", + "paths = subprocess.check_output(cmd,shell=True).decode('utf-8').split()", + "sys.path.extend(paths)", + "symbols_loaded = False", + "try:", + " if gdb.objfiles()[0].lookup_global_symbol('main') is not None:", + " symbols_loaded = True", + "except Exception:", + " pass", + "if not symbols_loaded:", + r" gdb.write('Could not find main symbol, debug info may not be loaded.\n')", + r" gdb.write('TEST FAILED -- No Symbols.\\\n')", + " gdb.execute('quit 1', to_string=True)", + "else:", + r" gdb.write('Symbols loaded.\n')", + "gdb.execute('set confirm off')", + "gdb.execute('source .gdbinit')", + "try:", + " verify_requirements(executable='python3')", + "except MissingRequirements as ex:", + " print(ex)", + " print('continuing testing anyways!')", + "except Exception as exc:", + " print('ERROR: failed while verifying requirements.')", + " traceback.print_exc()", + " sys.exit(1)", +] + +args = parser.parse_args() + +with open(args.pretty_printer_output, "w") as pretty_printer_output: + with open(args.pip_requirements_script) as pip_requirements_script: + pretty_printer_output.write(pip_requirements_script.read()) + pretty_printer_output.write("\n") + pretty_printer_output.write("\n".join(extra_lines)) + pretty_printer_output.write("\n") + with open(args.gdb_test_script) as test_script: + pretty_printer_output.write(test_script.read()) + +# Verbose and test_args are hardcoded because there are no users of those overrides currently +replacements = { + "@VERBOSE@": "True", + "@pretty_printer_test_py@": args.final_pretty_printer_path, + "@gdb_path@": args.gdb_path, + "@pretty_printer_test_program@": args.final_binary_path, + "@test_args@": '[""]', +} + +escaped_replacements = dict((re.escape(k), v) for k, v in replacements.items()) +pattern = re.compile("|".join(escaped_replacements.keys())) + +with open(args.pretty_printer_launcher_output, "w") as pretty_printer_launcher_output: + with open(args.pretty_printer_launcher_infile) as pretty_printer_launcher_infile: + replaced_text = pattern.sub( + lambda match: escaped_replacements[re.escape(match.group(0))], + pretty_printer_launcher_infile.read(), + ) + pretty_printer_launcher_output.write(replaced_text) diff --git a/bazel/install_rules/pretty_printer_tests.bzl b/bazel/install_rules/pretty_printer_tests.bzl new file mode 100644 index 00000000000..5a5eac47a8d --- /dev/null +++ b/bazel/install_rules/pretty_printer_tests.bzl @@ -0,0 +1,78 @@ +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") +load("//bazel/install_rules:providers.bzl", "TestBinaryInfo") + +# This will not currently work under bazel test/run until we have a version of gdb to use in bazel +def mongo_pretty_printer_test_impl(ctx): + split_name = ctx.label.name.split("-") + + # eg optimizer_gdb_test + short_name = split_name[-1] + + # eg /install-dist-test/bin/ + final_output_directory = "/" + "-".join(split_name[:-1]) + "/bin/" + + runnable_binary = None + for file in ctx.attr.test_binary.files.to_list(): + if file.extension == "" or file.extension == "exe": + runnable_binary = file + + pretty_printer_directory = "pretty-printer" + if "stripped" in split_name: + pretty_printer_directory += "-stripped" + pretty_printer_directory += "/" + + script_output = ctx.actions.declare_file(pretty_printer_directory + short_name + ".py") + launcher_output = ctx.actions.declare_file(pretty_printer_directory + "pretty_printer_test_launcher_" + short_name + ".py") + + python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime + + inputs = depset(transitive = [ + ctx.attr._pretty_printer_creation_script.files, + ctx.attr.test_script.files, + ctx.attr._pip_requirements_script.files, + ctx.attr._pretty_printer_launcher_infile.files, + python.files, + ]) + outputs = [launcher_output, script_output] + + ctx.actions.run( + executable = python.interpreter.path, + outputs = outputs, + inputs = inputs, + arguments = [ + ctx.file._pretty_printer_creation_script.path, + "--gdb-test-script=" + ctx.file.test_script.path, + "--pip-requirements-script=" + ctx.file._pip_requirements_script.path, + "--pretty-printer-output=" + script_output.path, + "--pretty-printer-launcher-infile=" + ctx.file._pretty_printer_launcher_infile.path, + # TODO have a way to get to gdb from inside bazel + "--gdb-path=" + "/opt/mongodbtoolchain/v4/bin/gdb", + # This is due to us being dependent on the final location of installed binaries - ideally we don't do this and run the tests + # in place and not from another directory + "--final-binary-path=" + ctx.bin_dir.path + final_output_directory + runnable_binary.basename, + "--final-pretty-printer-path=" + ctx.bin_dir.path + final_output_directory + script_output.basename, + "--pretty-printer-launcher-output=" + launcher_output.path, + ], + mnemonic = "MongoPrettyPrinterTestCreation", + ) + + default_provider = DefaultInfo(executable = launcher_output, files = depset(outputs)) + test_binary_provider = TestBinaryInfo(test_binaries = depset([launcher_output])) + return [default_provider, test_binary_provider] + +mongo_pretty_printer_test = rule( + mongo_pretty_printer_test_impl, + attrs = { + "test_script": attr.label(allow_single_file = True), + "test_binary": attr.label(), + # TODO have a way to get to gdb from inside bazel + #"_gdb": attr.label(allow_single_file = True, default = "//:gdb"), + "_pretty_printer_creation_script": attr.label(allow_single_file = True, default = "//bazel/install_rules:pretty_printer_test_creator.py"), + "_pip_requirements_script": attr.label(allow_single_file = True, default = "//site_scons/mongo:pip_requirements.py"), + "_pretty_printer_launcher_infile": attr.label(allow_single_file = True, default = "//src/mongo/util:pretty_printer_test_launcher.py.in"), + }, + doc = "Create pretty printer tests", + toolchains = ["@bazel_tools//tools/python:toolchain_type"], + executable = True, + test = True, +) diff --git a/bazel/install_rules/providers.bzl b/bazel/install_rules/providers.bzl new file mode 100644 index 00000000000..aa0df4d0762 --- /dev/null +++ b/bazel/install_rules/providers.bzl @@ -0,0 +1,6 @@ +TestBinaryInfo = provider( + doc = "Test binaries returned by this target.", + fields = { + "test_binaries": "group of all test binaries", + }, +) diff --git a/site_scons/mongo/BUILD.bazel b/site_scons/mongo/BUILD.bazel new file mode 100644 index 00000000000..13eb083863d --- /dev/null +++ b/site_scons/mongo/BUILD.bazel @@ -0,0 +1,3 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files(["pip_requirements.py"]) diff --git a/src/mongo/db/concurrency/BUILD.bazel b/src/mongo/db/concurrency/BUILD.bazel index cfd63ac4e8c..5fd6d6e8bec 100644 --- a/src/mongo/db/concurrency/BUILD.bazel +++ b/src/mongo/db/concurrency/BUILD.bazel @@ -6,6 +6,7 @@ exports_files( glob([ "*.h", "*.cpp", + "*.py", ]), ) diff --git a/src/mongo/db/query/optimizer/BUILD.bazel b/src/mongo/db/query/optimizer/BUILD.bazel index 0e6bd090d85..bd0d630f806 100644 --- a/src/mongo/db/query/optimizer/BUILD.bazel +++ b/src/mongo/db/query/optimizer/BUILD.bazel @@ -6,6 +6,7 @@ exports_files( glob([ "*.h", "*.cpp", + "*.py", ]), ) diff --git a/src/mongo/util/BUILD.bazel b/src/mongo/util/BUILD.bazel index d717126e52f..8c40728b7c8 100644 --- a/src/mongo/util/BUILD.bazel +++ b/src/mongo/util/BUILD.bazel @@ -5,7 +5,12 @@ load("//bazel/config:render_template.bzl", "render_template") package(default_visibility = ["//visibility:public"]) exports_files( - glob(["*.h"]) + glob(["*.cpp"]), + glob([ + "*.h", + "*.cpp", + "*.py", + "*.in", + ]), ) # Headers designed to be used without source code. Split these out