From c0d93b13743f772c4dcd78fbcb434c9707752293 Mon Sep 17 00:00:00 2001 From: Steve McClure Date: Tue, 19 Aug 2025 16:52:34 -0400 Subject: [PATCH] SERVER-108972: Support loading resmoke libs and suites from module directories (#40272) GitOrigin-RevId: c95ce37e0fa67f2626874c61ec076d740f184b8c --- buildscripts/ciconfig/evergreen.py | 2 +- .../resmokeconfig/resmoke_modules.yml | 12 ++++ buildscripts/resmokelib/config.py | 16 +++++- buildscripts/resmokelib/configure_resmoke.py | 55 ++++++++++++++++++- .../resmokelib/mongo_fuzzer_configs.py | 2 +- buildscripts/resmokelib/run/__init__.py | 39 ++++++++----- buildscripts/resmokelib/selector.py | 35 ++++++------ buildscripts/resmokelib/suitesconfig.py | 17 +++--- buildscripts/tests/ciconfig/evergreen.yml | 2 +- buildscripts/tests/ciconfig/test_evergreen.py | 10 ++-- .../resmoke_end2end/fixtures/BUILD.bazel | 6 ++ .../resmoke_end2end/fixtures/test_fixture.py | 6 ++ .../tests/resmoke_end2end/hooks/BUILD.bazel | 6 ++ .../tests/resmoke_end2end/hooks/test_hook.py | 5 ++ .../suites/resmoke_test_module_jstests.yml | 10 ++++ .../suites/resmoke_test_module_worked.yml | 11 ++++ .../tests/resmoke_end2end/test_resmoke.py | 45 ++++++++++++++- .../resmoke_end2end/test_resmoke_modules.yml | 9 +++ docs/testing/resmoke_modules.md | 33 +++++++++++ .../variants/amazon/test_dev.yml | 2 +- .../variants/amazon/test_release.yml | 8 +-- .../variants/debian/test_release.yml | 2 +- .../variants/macos/test_release.yml | 4 +- .../variants/mongot/test_dev.yml | 2 +- .../variants/rhel/test_release.yml | 10 ++-- .../variants/sanitizer/test_dev.yml | 4 +- .../sanitizer/test_dev_master_branch_only.yml | 4 +- .../variants/suse/test_release.yml | 2 +- .../variants/ubuntu/test_release.yml | 12 ++-- .../variants/windows/test_release.yml | 2 +- .../test_dev_master_branch_only.yml | 2 +- 31 files changed, 294 insertions(+), 81 deletions(-) create mode 100644 buildscripts/resmokeconfig/resmoke_modules.yml create mode 100644 buildscripts/tests/resmoke_end2end/fixtures/BUILD.bazel create mode 100644 buildscripts/tests/resmoke_end2end/fixtures/test_fixture.py create mode 100644 buildscripts/tests/resmoke_end2end/hooks/BUILD.bazel create mode 100644 buildscripts/tests/resmoke_end2end/hooks/test_hook.py create mode 100644 buildscripts/tests/resmoke_end2end/suites/resmoke_test_module_jstests.yml create mode 100644 buildscripts/tests/resmoke_end2end/suites/resmoke_test_module_worked.yml create mode 100644 buildscripts/tests/resmoke_end2end/test_resmoke_modules.yml create mode 100644 docs/testing/resmoke_modules.md diff --git a/buildscripts/ciconfig/evergreen.py b/buildscripts/ciconfig/evergreen.py index 47b1e48df23..633e12d07e4 100644 --- a/buildscripts/ciconfig/evergreen.py +++ b/buildscripts/ciconfig/evergreen.py @@ -345,7 +345,7 @@ class Variant(object): def is_enterprise_build(self) -> bool: """Determine if the build variant is configured for enterprise builds.""" - pattern = r"--enableEnterpriseTests\s*=?\s*off" + pattern = r"--modules\s*=?\s*none" return not re.search(pattern, str(self.raw)) @property diff --git a/buildscripts/resmokeconfig/resmoke_modules.yml b/buildscripts/resmokeconfig/resmoke_modules.yml new file mode 100644 index 00000000000..1980ef145ac --- /dev/null +++ b/buildscripts/resmokeconfig/resmoke_modules.yml @@ -0,0 +1,12 @@ +enterprise: + jstest_dirs: + - src/mongo/db/modules/enterprise/jstests +# atlas: +# fixture_dirs: +# - buildscripts/modules/atlas/fixtures +# hook_dirs: +# - buildscripts/modules/atlas/hooks +# suite_dirs: +# - buildscripts/modules/atlas/suites +# jstest_dirs: +# - buildscripts/modules/atlas/jstests diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 577b459562b..fc73ba332fc 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -120,7 +120,8 @@ DEFAULTS = { "shuffle": None, "stagger_jobs": None, "majority_read_concern": "on", - "enable_enterprise_tests": "on", + "modules": "default", + "resmoke_modules_path": os.path.join("buildscripts", "resmokeconfig", "resmoke_modules.yml"), "enable_evergreen_api_test_selection": False, "test_selection_strategies_array": None, "mongo_version_file": None, @@ -362,8 +363,17 @@ DRY_RUN = None # If set, specifies which node is the config shard. Can also be set to 'any'. CONFIG_SHARD = None -# if set, enables enterprise jstest to automatically be included -ENABLE_ENTERPRISE_TESTS = None +# path for the resmoke module config, should only be changed in testing. +MODULES_CONFIG_PATH = None + +# list of enabled modules +MODULES = None + +# list of dirs from enabled modules to get suites from +MODULE_SUITE_DIRS = [] + +# list of dirs from disabled modules to filter out of suite selectors +MODULE_DISABLED_JSTEST_DIRS = [] # if set, enables test selection using the Evergreen API ENABLE_EVERGREEN_API_TEST_SELECTION = None diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index 8f86db85b93..6c414acec6a 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -14,6 +14,7 @@ import subprocess import sys import textwrap import traceback +from functools import cache from pathlib import Path from typing import Dict, Optional @@ -29,6 +30,7 @@ from buildscripts.idl import gen_all_feature_flag_list from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import mongo_fuzzer_configs, multiversionsetupconstants, utils from buildscripts.resmokelib.run import TestRunner +from buildscripts.resmokelib.utils import autoloader from buildscripts.resmokelib.utils.batched_baggage_span_processor import BatchedBaggageSpanProcessor from buildscripts.resmokelib.utils.file_span_exporter import FileSpanExporter from buildscripts.util.read_config import read_config_file @@ -52,6 +54,7 @@ def validate_and_update_config( _update_config_vars(parser, args, should_configure_otel) _update_symbolizer_secrets() _validate_config(parser) + _set_up_modules() _set_logging_config() @@ -59,6 +62,45 @@ def process_feature_flag_file(path: str) -> list[str]: with open(path) as fd: return fd.read().split() +@cache +def _get_module_configs() -> dict: + with open(_config.MODULES_CONFIG_PATH, "r") as file: + return yaml.safe_load(file) + +def _set_up_modules(): + module_configs = _get_module_configs() + # verify all enabled modules exist + for module in _config.MODULES: + if module not in module_configs: + raise RuntimeError(f"Could not find configuration for module {module}") + + # loop through all modules, we need to act on both enabled and disabled modules + for module in module_configs.keys(): + module_config = module_configs[module] + all_paths_present = True + for key in ("fixture_dirs", "hook_dirs", "suite_dirs", "jstest_dirs"): + if key not in module_config: + continue + assert type(module_config[key]) == list, f"{key} in {module} did not have the expected type of list" + for module_dir in module_config[key]: + if not os.path.exists(module_dir): + all_paths_present = False + break + + + if module in _config.MODULES and all_paths_present: + # both the fixures and the hooks just need to be loaded once for resmoke to recognize them + for resource_dir in module_config.get("fixture_dirs", []) + module_config.get("hook_dirs", []): + norm_path = os.path.normpath(resource_dir) + package = norm_path.replace("/", ".") + autoloader.load_all_modules(package, [norm_path]) + + for suite_dir in module_config.get("suite_dirs", []): + _config.MODULE_SUITE_DIRS.append(suite_dir) + else: + for jstest_dir in module_config.get("jstest_dirs", []): + _config.MODULE_DISABLED_JSTEST_DIRS.append(jstest_dir) + def _validate_options(parser: argparse.ArgumentParser, args: dict): """Do preliminary validation on the options and error on any invalid options.""" @@ -497,7 +539,18 @@ flags in common: {common_set} _config.JOBS = config.pop("jobs") _config.LINEAR_CHAIN = config.pop("linear_chain") == "on" _config.MAJORITY_READ_CONCERN = config.pop("majority_read_concern") == "on" - _config.ENABLE_ENTERPRISE_TESTS = config.pop("enable_enterprise_tests") + _config.MODULES_CONFIG_PATH = config.pop("resmoke_modules_path") + modules = config.pop("modules").strip() + if modules == "default": + # try turning on all modules + _config.MODULES = list(_get_module_configs().keys()) + elif modules == "none": + _config.MODULES = [] + elif not modules: + raise RuntimeError("--modules must contain content. Specify 'none' to define no modules") + else: + _config.MODULES = modules.split(",") + _config.MIXED_BIN_VERSIONS = config.pop("mixed_bin_versions") if _config.MIXED_BIN_VERSIONS is not None: _config.MIXED_BIN_VERSIONS = _config.MIXED_BIN_VERSIONS.split("-") diff --git a/buildscripts/resmokelib/mongo_fuzzer_configs.py b/buildscripts/resmokelib/mongo_fuzzer_configs.py index d73a90c6340..bdc468acb60 100644 --- a/buildscripts/resmokelib/mongo_fuzzer_configs.py +++ b/buildscripts/resmokelib/mongo_fuzzer_configs.py @@ -186,7 +186,7 @@ def generate_encryption_config(rng: random.Random): # encryption also required an enterprise binary. if ( config.STORAGE_ENGINE != "wiredTiger" - or config.ENABLE_ENTERPRISE_TESTS != "on" + or "enterprise" not in config.MODULES or config.DISABLE_ENCRYPTION_FUZZING ): return ret diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index 4eb8d60ffc6..ba7d43d9018 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -105,7 +105,9 @@ class TestRunner(Subcommand): elif self.__command == "generate-matrix-suites": suitesconfig.generate() elif config.DRY_RUN == "tests": - self.dry_run() + self.dry_run(only_included=False) + elif config.DRY_RUN == "included-tests": + self.dry_run(only_included=True) else: self.run_tests() finally: @@ -183,15 +185,16 @@ class TestRunner(Subcommand): memberships[test] = test_membership[test] return memberships - def dry_run(self): + def dry_run(self, only_included=False): """List which tests would run and which tests would be excluded in a resmoke invocation.""" suites = self._get_suites() for suite in suites: self._shuffle_tests(suite) sb = ["Tests that would be run in suite {}".format(suite.get_display_name())] sb.extend(suite.tests or ["(no tests)"]) - sb.append("Tests that would be excluded from suite {}".format(suite.get_display_name())) - sb.extend(suite.excluded or ["(no tests)"]) + if not only_included: + sb.append("Tests that would be excluded from suite {}".format(suite.get_display_name())) + sb.extend(suite.excluded or ["(no tests)"]) self._exec_logger.info("\n".join(sb)) def run_tests(self): @@ -1453,7 +1456,7 @@ class RunPlugin(PluginInterface): "--dryRun", action="store", dest="dry_run", - choices=("off", "tests"), + choices=("off", "tests", "included-tests"), metavar="MODE", help=( "Instead of running the tests, outputs the tests that would be run" @@ -1839,6 +1842,13 @@ class RunPlugin(PluginInterface): dest="releases_file", help="An explicit releases YAML, if you do not want resmoke to fetch the latest one automatically.", ) + parser.add_argument( + "--modules", + dest="modules", + type=str, + help="Comma separated list of modules enabled, by default all modules that exist on dist will be enabled.", + default="default", + ) configure_resmoke.add_otel_args(parser) @@ -1914,16 +1924,6 @@ class RunPlugin(PluginInterface): help=("Enable or disable majority read concern support." " Defaults to %(default)s."), ) - mongodb_server_options.add_argument( - "--enableEnterpriseTests", - action="store", - dest="enable_enterprise_tests", - default="on", - choices=("on", "off"), - metavar="ON|OFF", - help=("Enable or disable enterprise tests. Defaults to 'on'."), - ) - mongodb_server_options.add_argument( "--storageEngine", dest="storage_engine", @@ -2160,6 +2160,15 @@ class RunPlugin(PluginInterface): " to allow for easier debugging" ), ) + + internal_options.add_argument( + "--resmokeModulesPath", + dest="resmoke_modules_path", + type=str, + help=( + "Sets the path to the resmoke modules config to allow testing different configurations." + ), + ) evergreen_options = parser.add_argument_group( title=_EVERGREEN_ARGUMENT_TITLE, diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py index 32a96f62775..c2c67031230 100644 --- a/buildscripts/resmokelib/selector.py +++ b/buildscripts/resmokelib/selector.py @@ -17,8 +17,6 @@ import buildscripts.resmokelib.testing.tags as _tags from buildscripts.resmokelib import config, errors, utils from buildscripts.resmokelib.utils import globstar, jscomment -ENTERPRISE_TEST_DIR = os.path.normpath("src/mongo/db/modules/enterprise/jstests") - ######################## # Test file explorer # ######################## @@ -294,13 +292,22 @@ class _TestList(object): for path in paths.evaluated: self._filtered.discard(path) - def filter_enterprise_tests(self): - """Exclude tests that start with the enterprise module directory from the test list.""" - self._filtered = { - test - for test in self._filtered - if not os.path.normpath(test).startswith(ENTERPRISE_TEST_DIR) - } + def filter_module_tests(self): + """Exclude tests that start directories of disabled module jstest dirs.""" + if not config.MODULE_DISABLED_JSTEST_DIRS: + return + + new_filtered = [] + for test in self._filtered: + in_disabled_module = False + for disabled_jstest_dir in config.MODULE_DISABLED_JSTEST_DIRS: + if os.path.normpath(test).startswith(os.path.normpath(disabled_jstest_dir)): + in_disabled_module = True + break + if not in_disabled_module: + new_filtered.append(test) + + self._filtered = new_filtered def match_tag_expression(self, tag_expression, get_tags): """Filter the test list to only include tests that match the tag expression. @@ -561,9 +568,8 @@ class _Selector(object): # 5. Apply the include files last with force=True to take precedence over the tags. if self._tests_are_files and selector_config.include_files: test_list.include_files(selector_config.include_files) - # 6: Apply the enterprise tests filter - if self.get_enterprise_tests_status() == "off": - test_list.filter_enterprise_tests() + # 6: Filter tests in disabled modules + test_list.filter_module_tests() return self.sort_tests(*test_list.get_tests()) @@ -590,11 +596,6 @@ class _Selector(object): """Retrieve the tags associated with the give test file.""" return [] - @staticmethod - def get_enterprise_tests_status() -> str: - """Get the status of enterprise tests from the configuration.""" - return config.ENABLE_ENTERPRISE_TESTS - class _JSTestSelectorConfig(_SelectorConfig): """_SelectorConfig subclass for JavaScript tests.""" diff --git a/buildscripts/resmokelib/suitesconfig.py b/buildscripts/resmokelib/suitesconfig.py index 0d2f9ed0a6b..995640c3f6f 100644 --- a/buildscripts/resmokelib/suitesconfig.py +++ b/buildscripts/resmokelib/suitesconfig.py @@ -273,14 +273,15 @@ class ExplicitSuiteConfig(SuiteConfigInterface): """Populate the named suites by scanning config_dir/suites.""" with cls._name_suites_lock: if not cls._named_suites: - suites_dir = os.path.join(_config.CONFIG_DIR, "suites") - root = os.path.abspath(suites_dir) - files = os.listdir(root) - for filename in files: - (short_name, ext) = os.path.splitext(filename) - if ext in (".yml", ".yaml"): - pathname = os.path.join(root, filename) - cls._named_suites[short_name] = pathname + suites_dirs = [os.path.join(_config.CONFIG_DIR, "suites")] + _config.MODULE_SUITE_DIRS + for suites_dir in suites_dirs: + root = os.path.abspath(suites_dir) + files = os.listdir(root) + for filename in files: + (short_name, ext) = os.path.splitext(filename) + if ext in (".yml", ".yaml"): + pathname = os.path.join(root, filename) + cls._named_suites[short_name] = pathname return cls._named_suites diff --git a/buildscripts/tests/ciconfig/evergreen.yml b/buildscripts/tests/ciconfig/evergreen.yml index e1cc85346e9..cc0b0e88eb6 100644 --- a/buildscripts/tests/ciconfig/evergreen.yml +++ b/buildscripts/tests/ciconfig/evergreen.yml @@ -183,7 +183,7 @@ buildvariants: run_on: - ubuntu1404-test expansions: - test_flags: "--param=value --ubuntu --enableEnterpriseTests=off" + test_flags: "--param=value --ubuntu --modules=none" tasks: - name: compile - name: passing_test diff --git a/buildscripts/tests/ciconfig/test_evergreen.py b/buildscripts/tests/ciconfig/test_evergreen.py index 327682e303c..032dada6263 100644 --- a/buildscripts/tests/ciconfig/test_evergreen.py +++ b/buildscripts/tests/ciconfig/test_evergreen.py @@ -478,7 +478,7 @@ class TestVariant(unittest.TestCase): def test_expansion(self): variant_ubuntu = self.conf.get_variant("ubuntu") self.assertEqual( - "--param=value --ubuntu --enableEnterpriseTests=off", + "--param=value --ubuntu --modules=none", variant_ubuntu.expansion("test_flags"), ) self.assertEqual(None, variant_ubuntu.expansion("not_a_valid_expansion_name")) @@ -486,7 +486,7 @@ class TestVariant(unittest.TestCase): def test_expansions(self): variant_ubuntu = self.conf.get_variant("ubuntu") self.assertEqual( - {"test_flags": "--param=value --ubuntu --enableEnterpriseTests=off"}, + {"test_flags": "--param=value --ubuntu --modules=none"}, variant_ubuntu.expansions, ) @@ -519,7 +519,7 @@ class TestVariant(unittest.TestCase): def test_test_flags(self): variant_ubuntu = self.conf.get_variant("ubuntu") self.assertEqual( - "--param=value --ubuntu --enableEnterpriseTests=off", variant_ubuntu.test_flags + "--param=value --ubuntu --modules=none", variant_ubuntu.test_flags ) variant_osx = self.conf.get_variant("osx-108") @@ -551,7 +551,7 @@ class TestVariant(unittest.TestCase): resmoke_task = variant_ubuntu.get_task("resmoke_task") self.assertEqual( { - "resmoke_task": "--suites=resmoke_task --storageEngine=wiredTiger --param=value --ubuntu --enableEnterpriseTests=off" + "resmoke_task": "--suites=resmoke_task --storageEngine=wiredTiger --param=value --ubuntu --modules=none" }, resmoke_task.combined_suite_to_resmoke_args_map, ) @@ -560,7 +560,7 @@ class TestVariant(unittest.TestCase): passing_task = variant_ubuntu.get_task("passing_test") self.assertEqual( { - "passing_test": "--suites=passing_test --param=value --ubuntu --enableEnterpriseTests=off" + "passing_test": "--suites=passing_test --param=value --ubuntu --modules=none" }, passing_task.combined_suite_to_resmoke_args_map, ) diff --git a/buildscripts/tests/resmoke_end2end/fixtures/BUILD.bazel b/buildscripts/tests/resmoke_end2end/fixtures/BUILD.bazel new file mode 100644 index 00000000000..fa7edff1bc5 --- /dev/null +++ b/buildscripts/tests/resmoke_end2end/fixtures/BUILD.bazel @@ -0,0 +1,6 @@ +# TODO(SERVER-105817): The following library is autogenerated, please split these out into individual python targets +py_library( + name = "all_python_files", + srcs = glob(["*.py"]), + visibility = ["//visibility:public"], +) diff --git a/buildscripts/tests/resmoke_end2end/fixtures/test_fixture.py b/buildscripts/tests/resmoke_end2end/fixtures/test_fixture.py new file mode 100644 index 00000000000..eb95aa6ec36 --- /dev/null +++ b/buildscripts/tests/resmoke_end2end/fixtures/test_fixture.py @@ -0,0 +1,6 @@ +from buildscripts.resmokelib.testing.fixtures.standalone import MongoDFixture + + +# just a copy of MongoDFixture for testing +class MongoDFixtureTesting(MongoDFixture): + REGISTERED_NAME = "MongoDFixtureTesting" diff --git a/buildscripts/tests/resmoke_end2end/hooks/BUILD.bazel b/buildscripts/tests/resmoke_end2end/hooks/BUILD.bazel new file mode 100644 index 00000000000..fa7edff1bc5 --- /dev/null +++ b/buildscripts/tests/resmoke_end2end/hooks/BUILD.bazel @@ -0,0 +1,6 @@ +# TODO(SERVER-105817): The following library is autogenerated, please split these out into individual python targets +py_library( + name = "all_python_files", + srcs = glob(["*.py"]), + visibility = ["//visibility:public"], +) diff --git a/buildscripts/tests/resmoke_end2end/hooks/test_hook.py b/buildscripts/tests/resmoke_end2end/hooks/test_hook.py new file mode 100644 index 00000000000..b2daf92c550 --- /dev/null +++ b/buildscripts/tests/resmoke_end2end/hooks/test_hook.py @@ -0,0 +1,5 @@ +from buildscripts.resmokelib.testing.hooks.validate import ValidateCollections + + +class ValidateCollectionsTesting(ValidateCollections): + REGISTERED_NAME = "ValidateCollectionsTesting" diff --git a/buildscripts/tests/resmoke_end2end/suites/resmoke_test_module_jstests.yml b/buildscripts/tests/resmoke_end2end/suites/resmoke_test_module_jstests.yml new file mode 100644 index 00000000000..f89760c0648 --- /dev/null +++ b/buildscripts/tests/resmoke_end2end/suites/resmoke_test_module_jstests.yml @@ -0,0 +1,10 @@ +test_kind: js_test + +selector: + roots: + - jstests/**/*.js + - buildscripts/tests/resmoke_end2end/testfiles/**/*.js + +executor: + fixture: + class: MongoDFixture diff --git a/buildscripts/tests/resmoke_end2end/suites/resmoke_test_module_worked.yml b/buildscripts/tests/resmoke_end2end/suites/resmoke_test_module_worked.yml new file mode 100644 index 00000000000..6157f136177 --- /dev/null +++ b/buildscripts/tests/resmoke_end2end/suites/resmoke_test_module_worked.yml @@ -0,0 +1,11 @@ +test_kind: js_test + +selector: + roots: + - buildscripts/tests/resmoke_end2end/testfiles/validatecollections/test_pass.js + +executor: + hooks: + - class: ValidateCollectionsTesting + fixture: + class: MongoDFixtureTesting diff --git a/buildscripts/tests/resmoke_end2end/test_resmoke.py b/buildscripts/tests/resmoke_end2end/test_resmoke.py index 3335c8f028a..c90aa6f83f7 100644 --- a/buildscripts/tests/resmoke_end2end/test_resmoke.py +++ b/buildscripts/tests/resmoke_end2end/test_resmoke.py @@ -12,6 +12,7 @@ import sys import time import unittest from shutil import rmtree +from typing import List import yaml @@ -613,9 +614,9 @@ class TestDiscovery(_ResmokeSelftest): ) -def execute_resmoke(resmoke_args): +def execute_resmoke(resmoke_args: List[str], subcommand: str="run"): return subprocess.run( - [sys.executable, "buildscripts/resmoke.py", "run"] + resmoke_args, + [sys.executable, "buildscripts/resmoke.py", subcommand] + resmoke_args, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -895,3 +896,43 @@ class TestValidateCollections(unittest.TestCase): expected = "collection validation failed" self.assertIn(expected, result.stdout) self.assertNotEqual(result.returncode, 0) + +class TestModules(unittest.TestCase): + def test_files_included(self): + # this suite uses a fixture and hook from the module so it will fail if they are not loaded + # it also uses a + resmoke_args = [ + "--resmokeModulesPath=buildscripts/tests/resmoke_end2end/test_resmoke_modules.yml", + "--suite=resmoke_test_module_worked", + ] + + result = execute_resmoke(resmoke_args) + self.assertEqual(result.returncode, 0) + + def test_jstests_excluded(self): + # this first command should not include any of the tests from the module + resmoke_args = [ + "--resmokeModulesPath=buildscripts/tests/resmoke_end2end/test_resmoke_modules.yml", + "--modules=none", + "--suite=buildscripts/tests/resmoke_end2end/suites/resmoke_test_module_jstests.yml", + "--dryRun=included-tests", + ] + + result_without_module = execute_resmoke(resmoke_args) + self.assertEqual(result_without_module.returncode, 0) + + # this second invocartion should include all of the base jstests and all of the module jstests. + resmoke_args = [ + "--resmokeModulesPath=buildscripts/tests/resmoke_end2end/test_resmoke_modules.yml", + "--modules=default", + "--suite=buildscripts/tests/resmoke_end2end/suites/resmoke_test_module_jstests.yml", + "--dryRun=included-tests", + ] + + result_with_module = execute_resmoke(resmoke_args) + self.assertEqual(result_with_module.returncode, 0) + + # assert the test is in the list of tests when the module is included + self.assertIn("buildscripts/tests/resmoke_end2end/testfiles/one.js", result_with_module.stdout) + # assert the test is not in the list of tests when the module is excluded + self.assertNotIn("buildscripts/tests/resmoke_end2end/testfiles/one.js", result_without_module.stdout) diff --git a/buildscripts/tests/resmoke_end2end/test_resmoke_modules.yml b/buildscripts/tests/resmoke_end2end/test_resmoke_modules.yml new file mode 100644 index 00000000000..489a38ae2b7 --- /dev/null +++ b/buildscripts/tests/resmoke_end2end/test_resmoke_modules.yml @@ -0,0 +1,9 @@ +testing: + fixture_dirs: + - buildscripts/tests/resmoke_end2end/fixtures + hook_dirs: + - buildscripts/tests/resmoke_end2end/hooks + suite_dirs: + - buildscripts/tests/resmoke_end2end/suites + jstest_dirs: + - buildscripts/tests/resmoke_end2end/testfiles diff --git a/docs/testing/resmoke_modules.md b/docs/testing/resmoke_modules.md new file mode 100644 index 00000000000..d38075c4258 --- /dev/null +++ b/docs/testing/resmoke_modules.md @@ -0,0 +1,33 @@ +# Resmoke Module Configuration + +This configuration allows additional modules to be added to Resmoke, providing more context about their associated directories. Modules can specify directories for fixtures, hooks, suites, and JavaScript tests, which Resmoke incorporates during its testing process. + +## Adding a New Module + +To add a new module to Resmoke, define the module name and specify its `fixture_dirs`, `hook_dirs`, `suite_dirs`, and `jstest_dirs` in the YAML configuration. Each field should be a list of directory paths. + +### Example YAML Configuration + +```yaml +my_new_module: + fixture_dirs: + - path/to/my_new_module/fixtures + hook_dirs: + - path/to/my_new_module/hooks + suite_dirs: + - path/to/my_new_module/suites + jstest_dirs: + - path/to/my_new_module/jstests +``` + +### Field Descriptions + +- **`fixture_dirs`**: Directories containing fixtures associated with the module. +- **`hook_dirs`**: Directories containing hooks associated with the module. +- **`suite_dirs`**: Directories containing suites with test configurations. +- **`jstest_dirs`**: Directories containing JavaScript tests specific to the module. This ensures module-specific tests are excluded from other suite configurations when the module is disabled. + +## Notes + +- Any suite can use jstests from any directory, when the module is enabled the configured jstest dirs does nothing. Only when the module is disabled does it filter out the tests that might be configured in a suite from a different module. +- Fields can be omitted or empty lists diff --git a/etc/evergreen_yml_components/variants/amazon/test_dev.yml b/etc/evergreen_yml_components/variants/amazon/test_dev.yml index 730114a44d1..f2298e8a482 100644 --- a/etc/evergreen_yml_components/variants/amazon/test_dev.yml +++ b/etc/evergreen_yml_components/variants/amazon/test_dev.yml @@ -353,7 +353,7 @@ buildvariants: --linkopt=-s test_flags: >- --excludeWithAnyTags=incompatible_with_amazon_linux,requires_external_data_source - --enableEnterpriseTests=off + --modules=none has_packages: false repo_edition: org large_distro_name: amazon2023-arm64-latest-small-m8g diff --git a/etc/evergreen_yml_components/variants/amazon/test_release.yml b/etc/evergreen_yml_components/variants/amazon/test_release.yml index 13c2fc0bfd0..81369bde1a0 100644 --- a/etc/evergreen_yml_components/variants/amazon/test_release.yml +++ b/etc/evergreen_yml_components/variants/amazon/test_release.yml @@ -13,7 +13,7 @@ buildvariants: expansions: test_flags: >- --excludeWithAnyTags=SERVER-34286,incompatible_with_amazon_linux,requires_external_data_source - --enableEnterpriseTests=off + --modules=none push_path: linux push_bucket: downloads.mongodb.org push_bucket_new: cdn-origin-mongodb-server-community @@ -111,7 +111,7 @@ buildvariants: expansions: test_flags: >- --excludeWithAnyTags=SERVER-34286,incompatible_with_amazon_linux,requires_external_data_source - --enableEnterpriseTests=off + --modules=none push_path: linux push_bucket: downloads.mongodb.org push_bucket_new: cdn-origin-mongodb-server-community @@ -215,7 +215,7 @@ buildvariants: --linkopt=-s test_flags: >- --excludeWithAnyTags=incompatible_with_amazon_linux,requires_external_data_source - --enableEnterpriseTests=off + --modules=none has_packages: true packager_script: packager.py packager_arch: x86_64 @@ -353,7 +353,7 @@ buildvariants: --linkopt=-s test_flags: >- --excludeWithAnyTags=incompatible_with_amazon_linux,requires_external_data_source - --enableEnterpriseTests=off + --modules=none has_packages: true packager_script: packager.py packager_arch: aarch64 diff --git a/etc/evergreen_yml_components/variants/debian/test_release.yml b/etc/evergreen_yml_components/variants/debian/test_release.yml index 0398e427c50..e3e47e73fdb 100644 --- a/etc/evergreen_yml_components/variants/debian/test_release.yml +++ b/etc/evergreen_yml_components/variants/debian/test_release.yml @@ -25,7 +25,7 @@ buildvariants: --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source,requires_updated_oscrypto - --enableEnterpriseTests=off + --modules=none multiversion_platform: debian12 multiversion_edition: targeted has_packages: true diff --git a/etc/evergreen_yml_components/variants/macos/test_release.yml b/etc/evergreen_yml_components/variants/macos/test_release.yml index 48f5d233545..484773c041c 100644 --- a/etc/evergreen_yml_components/variants/macos/test_release.yml +++ b/etc/evergreen_yml_components/variants/macos/test_release.yml @@ -14,7 +14,7 @@ buildvariants: expansions: test_flags: >- --excludeWithAnyTags=incompatible_with_macos,requires_external_data_source - --enableEnterpriseTests=off + --modules=none push_path: osx push_bucket: downloads.mongodb.org push_bucket_new: cdn-origin-mongodb-server-community @@ -64,7 +64,7 @@ buildvariants: expansions: test_flags: >- --excludeWithAnyTags=incompatible_with_macos,requires_external_data_source - --enableEnterpriseTests=off + --modules=none push_path: osx push_bucket: downloads.mongodb.org push_bucket_new: cdn-origin-mongodb-server-community diff --git a/etc/evergreen_yml_components/variants/mongot/test_dev.yml b/etc/evergreen_yml_components/variants/mongot/test_dev.yml index 2dc4bd4e82e..d28a01ad0fb 100644 --- a/etc/evergreen_yml_components/variants/mongot/test_dev.yml +++ b/etc/evergreen_yml_components/variants/mongot/test_dev.yml @@ -86,7 +86,7 @@ buildvariants: --build_enterprise=False test_flags: >- --excludeWithAnyTags=incompatible_with_amazon_linux,requires_external_data_source - --enableEnterpriseTests=off + --modules=none --runAllFeatureFlagTests has_packages: false multiversion_platform: amazon2023 diff --git a/etc/evergreen_yml_components/variants/rhel/test_release.yml b/etc/evergreen_yml_components/variants/rhel/test_release.yml index 28a5f397738..e80408d3a51 100644 --- a/etc/evergreen_yml_components/variants/rhel/test_release.yml +++ b/etc/evergreen_yml_components/variants/rhel/test_release.yml @@ -20,7 +20,7 @@ buildvariants: build_mongoreplay: true test_flags: >- --excludeWithAnyTags=requires_os_access - --enableEnterpriseTests=off + --modules=none compile_variant: ubi8 tasks: - name: compile_and_archive_dist_test_TG @@ -51,7 +51,7 @@ buildvariants: --build_enterprise=False test_flags: >- --excludeWithAnyTags=requires_external_data_source,requires_increased_memlock_limits - --enableEnterpriseTests=off + --modules=none multiversion_platform: rhel8 multiversion_edition: targeted has_packages: true @@ -158,7 +158,7 @@ buildvariants: --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source,requires_increased_memlock_limits - --enableEnterpriseTests=off + --modules=none has_packages: true packager_script: packager.py packager_arch: aarch64 @@ -252,7 +252,7 @@ buildvariants: --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source - --enableEnterpriseTests=off + --modules=none multiversion_platform: rhel93 multiversion_edition: targeted has_packages: true @@ -350,7 +350,7 @@ buildvariants: --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source - --enableEnterpriseTests=off + --modules=none has_packages: true packager_script: packager.py packager_arch: aarch64 diff --git a/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml b/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml index 18806c95cbc..a95aaea8fcb 100644 --- a/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml +++ b/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml @@ -281,7 +281,7 @@ buildvariants: resmoke_jobs_factor: 0.5 # Avoid starting too many mongod's test_flags: >- --excludeWithAnyTags=requires_http_client - --enableEnterpriseTests=off + --modules=none --mongodSetParameters="{internalQueryEnableAggressiveSpillsInGroup: true}" target_resmoke_time: 15 max_sub_suites: 5 @@ -420,7 +420,7 @@ buildvariants: hang_analyzer_dump_core: false test_flags: >- --excludeWithAnyTags=requires_fast_memory,corrupts_data - --enableEnterpriseTests=off + --modules=none compile_variant: rhel8-asan tasks: - name: compile_test_serial_TG diff --git a/etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml index d0efbe763d0..0f233823fcb 100644 --- a/etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml @@ -266,7 +266,7 @@ buildvariants: large_distro_name: windows-2022-large test_flags: >- --excludeWithAnyTags=incompatible_with_windows_tls - --enableEnterpriseTests=off + --modules=none --mongodSetParameters="{internalQueryEnableAggressiveSpillsInGroup: true}" compile_variant: *windows-debug-suggested tasks: @@ -335,7 +335,7 @@ buildvariants: compile_variant: *macos-debug-suggested test_flags: >- --excludeWithAnyTags=incompatible_with_macos - --enableEnterpriseTests=off + --modules=none --mongodSetParameters="{internalQueryEnableAggressiveSpillsInGroup: true}" resmoke_jobs_max: 6 bazel_compile_flags: >- diff --git a/etc/evergreen_yml_components/variants/suse/test_release.yml b/etc/evergreen_yml_components/variants/suse/test_release.yml index 9a88409dd9f..15e2cf489fe 100644 --- a/etc/evergreen_yml_components/variants/suse/test_release.yml +++ b/etc/evergreen_yml_components/variants/suse/test_release.yml @@ -75,7 +75,7 @@ buildvariants: --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source,requires_increased_memlock_limits - --enableEnterpriseTests=off + --modules=none multiversion_platform: suse15 multiversion_edition: targeted has_packages: true diff --git a/etc/evergreen_yml_components/variants/ubuntu/test_release.yml b/etc/evergreen_yml_components/variants/ubuntu/test_release.yml index 5abfcb4f580..6e27a173c9d 100644 --- a/etc/evergreen_yml_components/variants/ubuntu/test_release.yml +++ b/etc/evergreen_yml_components/variants/ubuntu/test_release.yml @@ -26,7 +26,7 @@ buildvariants: --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source,requires_increased_memlock_limits - --enableEnterpriseTests=off + --modules=none multiversion_platform: ubuntu2204 multiversion_edition: targeted has_packages: true @@ -86,7 +86,7 @@ buildvariants: --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source,requires_increased_memlock_limits,requires_updated_oscrypto - --enableEnterpriseTests=off + --modules=none multiversion_platform: ubuntu2404 multiversion_edition: targeted has_packages: true @@ -145,7 +145,7 @@ buildvariants: --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source - --enableEnterpriseTests=off + --modules=none multiversion_platform: ubuntu2004 multiversion_edition: targeted has_packages: true @@ -404,7 +404,7 @@ buildvariants: --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source - --enableEnterpriseTests=off + --modules=none resmoke_jobs_max: 8 # Avoid starting too many mongod's on ARM test servers has_packages: true packager_script: packager.py @@ -503,7 +503,7 @@ buildvariants: --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source - --enableEnterpriseTests=off + --modules=none resmoke_jobs_max: 8 # Avoid starting too many mongod's on ARM test servers has_packages: true packager_script: packager.py @@ -607,7 +607,7 @@ buildvariants: --linkopt=-s test_flags: >- --excludeWithAnyTags=requires_external_data_source,requires_updated_oscrypto - --enableEnterpriseTests=off + --modules=none resmoke_jobs_max: 8 # Avoid starting too many mongod's on ARM test servers has_packages: true packager_script: packager.py diff --git a/etc/evergreen_yml_components/variants/windows/test_release.yml b/etc/evergreen_yml_components/variants/windows/test_release.yml index ffe016484a3..cf0983b3fd4 100644 --- a/etc/evergreen_yml_components/variants/windows/test_release.yml +++ b/etc/evergreen_yml_components/variants/windows/test_release.yml @@ -31,7 +31,7 @@ buildvariants: large_distro_name: windows-2022-large test_flags: >- --excludeWithAnyTags=incompatible_with_windows_tls,requires_external_data_source - --enableEnterpriseTests=off + --modules=none compile_variant: windows tasks: - name: compile_test_and_package_serial_TG diff --git a/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml index f12aff70460..ece5b50415e 100644 --- a/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml @@ -88,7 +88,7 @@ buildvariants: --build_enterprise=False test_flags: >- --excludeWithAnyTags=requires_http_client - --enableEnterpriseTests=off + --modules=none --mongodSetParameters="{internalQueryEnableAggressiveSpillsInGroup: true}" compile_variant: *linux_x86_dynamic_debug_wtdevelop_compile_variant_name tasks: