mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-94816 Add Resmoke run / test-discovery CLI option to calculate a subset of tests to use when more complex suite exists (#27943) 

GitOrigin-RevId: 13339393147d01adb577e2160fd671adda91900e
This commit is contained in:
Vishnu K 2024-10-21 17:59:20 -04:00 committed by MongoDB Bot
parent f30b35fd1a
commit 2bf91fdc39
9 changed files with 459 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
buildscripts/resmokelib/__init__.py
View File

@ -7,6 +7,7 @@ from buildscripts.resmokelib import (
parser,
reportfile,
sighandler,
suite_hierarchy,
suitesconfig,
testing,
utils,
@ -20,6 +21,7 @@ __all__ = [
"reportfile",
"sighandler",
"suitesconfig",
"suite_hierarchy",
"testing",
"utils",
]

4
buildscripts/resmokelib/config.py
View File

@ -66,6 +66,7 @@ DEFAULTS = {
"dry_run": None,
"exclude_with_any_tags": None,
"force_excluded_tests": False,
"skip_tests_covered_by_more_complex_suites": False,
"fuzz_mongod_configs": None,
"fuzz_runtime_params": None,
"fuzz_runtime_stress": "off",
@ -401,6 +402,9 @@ EXCLUDE_WITH_ANY_TAGS = None
# Allow test files passed as positional args to run even if they are excluded on the suite config.
FORCE_EXCLUDED_TESTS = None
# Only run tests on the given suite that will not be run on a more complex suite.
SKIP_TESTS_COVERED_BY_MORE_COMPLEX_SUITES = None
# A tag which is implicited excluded. This is useful for temporarily disabling a test.
EXCLUDED_TAG = "__TEMPORARILY_DISABLED__"

4
buildscripts/resmokelib/configure_resmoke.py
View File

@ -609,6 +609,10 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py
# Force invalid suite config
_config.FORCE_EXCLUDED_TESTS = config.pop("force_excluded_tests")
_config.SKIP_TESTS_COVERED_BY_MORE_COMPLEX_SUITES = config.pop(
"skip_tests_covered_by_more_complex_suites"
)
# Archival options. Archival is enabled only when running on evergreen.
if not _config.EVERGREEN_TASK_ID:
_config.ARCHIVE_FILE = None

9
buildscripts/resmokelib/discovery/__init__.py
View File

@ -99,6 +99,15 @@ class DiscoveryPlugin(PluginInterface):
TEST_DISCOVERY_SUBCOMMAND, help="Discover what tests are run by a suite."
)
parser.add_argument("--suite", metavar="SUITE", help="Suite to run against.")
parser.add_argument(
"--skipTestsCoveredByMoreComplexSuites",
dest="skip_tests_covered_by_more_complex_suites",
action="store_true",
help=(
"Excludes tests from running on some suite_A if a more complex"
" suite_A_B will also run the same tests."
),
)
parser = subparsers.add_parser(
SUITECONFIG_SUBCOMMAND, help="Display configuration of a test suite."

10
buildscripts/resmokelib/run/__init__.py
View File

@ -1210,6 +1210,16 @@ class RunPlugin(PluginInterface):
),
)
parser.add_argument(
"--skipTestsCoveredByMoreComplexSuites",
dest="skip_tests_covered_by_more_complex_suites",
action="store_true",
help=(
"Excludes tests from running on some suite_A if a more complex"
" suite_A_B will also run the same tests."
),
)
parser.add_argument(
"--genny",
dest="genny_executable",

159
buildscripts/resmokelib/suite_hierarchy.py Normal file
View File

@ -0,0 +1,159 @@
# Represents a DAG describing the complexity of suites. As we go deeper into the graph,
# suites get simpler. For example, suppose we have suite_A and suite_A_B, where suite_A_B
# tests a strict superset of *features* (not tests) tested by suite_A. Then, the graph
# would contain:
# {
# suite_A_B: {
# suite_A: {}
# }
# }
#
# But suppose feature_B changed the interaction with the server fundamentally - for example,
# if feature_B was retryable writes - then in this case suite_A_B would not be a strict superset
# of the features tested by suite_A. In this case, suite_A_B cannot be considered more complex
# than suite_A; the two are incomparable.
SUITE_HIERARCHY = {
# Concurrency suites
"concurrency": {},
"concurrency_compute_mode": {},
"concurrency_embedded_router_causal_consistency_and_balancer": {
"concurrency_embedded_router_causal_consistency": {}
},
"concurrency_embedded_router_clusterwide_ops_add_remove_shards": {},
"concurrency_embedded_router_local_read_write_multi_stmt_txn_with_balancer": {
"concurrency_embedded_router_local_read_write_multi_stmt_txn": {}
},
"concurrency_embedded_router_multi_stmt_txn_with_balancer": {
# concurrency_embedded_multi_stmt_txn is not considered a superset in terms
# of complexity of concurrency_embedded_router_replication because multi_stmt_txns
# are not a superset feature of regular operations.
"concurrency_embedded_router_multi_stmt_txn": {}
},
"concurrency_embedded_router_replication_with_balancer": {
"concurrency_embedded_router_replication": {},
},
"concurrency_multitenancy_replication_with_atlas_proxy": {},
"simulate_crash_concurrency_replication": {},
"concurrency_sharded_replication_with_balancer_and_config_transitions_and_add_remove_shard": {
"concurrency_sharded_with_balancer_and_auto_bootstrap": {}
},
"concurrency_sharded_replication_with_balancer_and_config_transitions": {
# The auto_bootstrap suites maintain a static config shard and so the config_transitions
# suite is a superset of it because it also transitions the config shard to a dedicated
# replica set.
"concurrency_sharded_with_balancer_and_auto_bootstrap": {
"concurrency_sharded_with_auto_bootstrap": {}
}
},
"concurrency_sharded_causal_consistency_and_balancer": {
"concurrency_sharded_causal_consistency": {}
},
"concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer": {
"concurrency_sharded_local_read_write_multi_stmt_txn": {}
},
"concurrency_sharded_multi_stmt_txn_with_balancer_and_config_transitions": {
"concurrency_sharded_multi_stmt_txn_with_balancer": {
"concurrency_sharded_multi_stmt_txn": {}
}
},
"concurrency_sharded_multi_stmt_txn_stepdown_terminate_kill_primary": {
"concurrency_sharded_multi_stmt_txn": {}
},
"concurrency_sharded_stepdown_terminate_kill_primary_with_balancer_and_config_transitions": {
"concurrency_sharded_stepdown_terminate_kill_primary_with_balancer": {
# The stepdown suite is not considered a superset of concurrency_sharded_replication
# because the stepdown suite uses retryable writes whereas the vanilla suite does not.
# Therefore the commands being sent to the server are fundamentally different.
"concurrency_sharded_with_stepdowns": {}
}
},
"concurrency_sharded_clusterwide_ops_add_remove_shards": {},
"concurrency_replication_causal_consistency": {},
"concurrency_replication_causal_consistency_with_replica_set_endpoint": {},
"concurrency_replication_for_backup_restore": {},
"concurrency_replication_for_export_import": {},
"concurrency_replication_multi_stmt_txn": {},
"concurrency_replication_multi_stmt_txn_with_replica_set_endpoint": {},
"concurrency_replication_with_replica_set_endpoint": {},
"concurrency_replication": {},
"concurrency_sharded_initial_sync": {"concurrency_sharded_causal_consistency": {}},
# JScore passthrough suites
}
def compute_dag(complexity_graph):
"""
Computes a graph of the below form from the nested complexity graph.
{
node: { parents: set(...), children: set(...) },
...
}
where the parents are the direct parents and children are direct
children. Note that if the original nested graph had multiple paths connecting
a node to another (A->B->C and A->C are both edges), then C would have both
B and A as its direct parents, and A would have B and C as its direct children
"""
graph = {}
# Initially place all the known ancestor nodes in the frontier.
frontier = []
for ancestor, descendants in complexity_graph.items():
frontier.append((ancestor, descendants))
while frontier:
parent, children = frontier.pop(0)
if parent not in graph:
graph[parent] = {"parents": set(), "children": set()}
for child, grandchildren in children.items():
if child not in graph:
graph[child] = {"parents": set(), "children": set()}
graph[parent]["children"].add(child)
graph[child]["parents"].add(parent)
frontier.append((child, grandchildren))
return graph
def compute_ancestors(node, dag):
"""Returns all the ancestor, direct and indirect, of the given node."""
ancestors = set()
frontier = [node]
while frontier:
node = frontier.pop(0)
ancestors = ancestors.union(dag[node]["parents"])
frontier += list(dag[node]["parents"])
return ancestors
def compute_minimal_test_set(suite_name, dag, tests_in_suite):
"""
Given a DAG that represents which suite is more complex than
another suite, and the set of tests that are usually run in each suite,
returns the minimal set of tests that need to be run for the given suite.
suite_name: the suite for which we want to calculate the minimal test set.
dag: a dictionary of dictionaries of the form:
{
"node_N" : {
"parents": set(["node_i", ...]),
"children": set(["node_k", ...])
}
}
tests_in_suite: a dictionary of suite_name -> set(tests) that can be run in the suite.
"""
# To calculate the minimal set for a suite 'curr_suite':
# 1) Figure out who all the ancestors of 'curr_suite' are.
# 2) Subtract from curr_suite's test set the union of the test sets of all its ancestors.
ancestors = compute_ancestors(suite_name, dag)
# Copy the given set
curr_test_set = set(tests_in_suite[suite_name])
for ancestor in ancestors:
curr_test_set -= tests_in_suite[ancestor]
return curr_test_set

49
buildscripts/resmokelib/suitesconfig.py
View File

@ -12,7 +12,7 @@ import yaml
import buildscripts.resmokelib.utils.filesystem as fs
from buildscripts.resmokelib import config as _config
from buildscripts.resmokelib import errors, utils
from buildscripts.resmokelib import errors, suite_hierarchy, utils
from buildscripts.resmokelib.logging import loggers
from buildscripts.resmokelib.testing import suite as _suite
from buildscripts.resmokelib.utils import load_yaml_file
@ -120,6 +120,7 @@ def get_suites(suite_names_or_paths: list[str], test_files: list[str]) -> List[_
for suite_filename in suite_names_or_paths:
suite_config = _get_suite_config(suite_filename)
suite = _suite.Suite(suite_filename, suite_config)
if suite_roots:
# Override the suite's default test files with those passed in from the command line.
override_suite_config = suite_config.copy()
@ -146,10 +147,53 @@ def get_suites(suite_names_or_paths: list[str], test_files: list[str]) -> List[_
f"'{test}' excluded in '{suite.get_name()}'"
)
suite = override_suite
if _config.SKIP_TESTS_COVERED_BY_MORE_COMPLEX_SUITES:
if suite_roots:
raise ValueError(
"Cannot use '--skipTestsCoveredByMoreComplexSuites' when tests have been passed in from the command line."
)
if _config.ORIGIN_SUITE:
raise ValueError(
"Cannot use '--originSuite' with '--skipTestsCoveredByMoreComplexSuites'."
)
suite = _compute_minimal_test_set_suite(suite_filename)
suites.append(suite)
return suites
def _compute_minimal_test_set_suite(origin_suite_name):
"""
Given a suite_A, returns the set of tests compatible with it, but incompatible
with any suite_A_B more complex than it.
The relationship of "more complex" is defined in suite_hierarchy.py.
"""
# Compute the dag from the complexity graph
dag = suite_hierarchy.compute_dag(suite_hierarchy.SUITE_HIERARCHY)
# If we don't know how to compute the minimal test set because the suite's
# information isn't present in the hierarchy, just return the suite as is.
if origin_suite_name not in dag:
suite_config = _get_suite_config(origin_suite_name)
suite = _suite.Suite(origin_suite_name, suite_config)
return suite
# Build a dictionary of the tests in each suite.
tests_in_suite = {}
for suite_in_dag in dag.keys():
suite_config = _get_suite_config(suite_in_dag)
suite = _suite.Suite(suite_in_dag, suite_config)
tests_in_suite[suite_in_dag] = set(suite.tests)
tests = suite_hierarchy.compute_minimal_test_set(origin_suite_name, dag, tests_in_suite)
min_suite_config = _get_suite_config(origin_suite_name).copy()
min_suite_config.update(_make_suite_roots(list(tests)))
return _suite.Suite(origin_suite_name, min_suite_config)
def _make_suite_roots(files):
return {"selector": {"roots": files}}
@ -527,5 +571,4 @@ class SuiteFinder(object):
def get_suite(suite_name_or_path) -> _suite.Suite:
"""Retrieve the Suite instance corresponding to a suite configuration file."""
suite_config = _get_suite_config(suite_name_or_path)
return _suite.Suite(suite_name_or_path, suite_config)
return get_suites([suite_name_or_path], None)[0]

224
buildscripts/tests/resmokelib/test_suite_hierarchy.py Normal file
View File

@ -0,0 +1,224 @@
"""Unit tests for buildscripts/resmokelib/suite_hierarchy.py."""
import unittest
from buildscripts.resmokelib.suite_hierarchy import (
compute_ancestors,
compute_dag,
compute_minimal_test_set,
)
tests_in_suite = {
"A": set(["t1", "t2", "t3"]),
"B": set(["t1", "t2", "t3", "t4"]),
"C": set(["t1", "t2", "t3", "t5"]),
"D": set(["t2", "t3", "t5", "t6"]),
"E": set(["t1", "t2", "t3", "t7"]),
"P": set(
[
"t1",
]
),
"Q": set(["t1", "t7"]),
}
# Graph 1:
# A P
# | |
# | V
# | Q
# | |
# V |
# B <-----+
# |
# +-------+
# | |
# V V
# C D
# |
# V
# E
#
# Different equivalent ways of representing Graph 1
graph1 = [
{"A": {"B": {"C": {"E": {}}, "D": {}}}, "P": {"Q": {"B": {}}}},
{
"A": {
"B": {},
},
"P": {"Q": {"B": {}}},
"B": {"C": {"E": {}}, "D": {}},
},
{"A": {"B": {"C": {"E": {}}, "D": {}}}, "P": {"Q": {}}, "Q": {"B": {}}},
{
"A": {"B": {}},
"B": {"C": {}, "D": {}},
"C": {"E": {}},
"D": {},
"P": {"Q": {}},
"Q": {"B": {}},
},
]
correct_dag1 = {
"A": {"parents": set(), "children": set(["B"])},
"B": {"parents": set(["A", "Q"]), "children": set(["C", "D"])},
"C": {"parents": set(["B"]), "children": set(["E"])},
"D": {"parents": set(["B"]), "children": set()},
"E": {"parents": set(["C"]), "children": set()},
"P": {"parents": set(), "children": set(["Q"])},
"Q": {"parents": set(["P"]), "children": set(["B"])},
}
correct_ancestors1 = [
("A", set()),
("B", set(["A", "P", "Q"])),
("C", set(["A", "P", "Q", "B"])),
("D", set(["A", "P", "Q", "B"])),
("E", set(["A", "P", "Q", "B", "C"])),
("P", set()),
("Q", set(["P"])),
]
# Minimal set for graph 1
correct_minimal_set1 = {
"A": set(["t1", "t2", "t3"]),
"B": set(["t4"]),
"C": set(["t5"]),
"D": set(["t5", "t6"]),
"E": set(),
"P": set(
[
"t1",
]
),
"Q": set(["t7"]),
}
# Graph 2:
# A--+ P (disconnected)
# | |
# | |
# | | Q (disconnected)
# | |
# V |
# B |
# | |
# | |
# | |
# V |
# C <+
#
# Different equivalent ways of representing Graph 2
graph2 = [
{"A": {"B": {"C": {}}, "C": {}}, "P": {}, "Q": {}},
{"B": {"C": {}}, "A": {"B": {}, "C": {}}, "P": {}, "Q": {}},
]
correct_dag2 = {
"A": {"parents": set(), "children": set(["B", "C"])},
"B": {"parents": set(["A"]), "children": set(["C"])},
"C": {"parents": set(["A", "B"]), "children": set()},
"P": {"parents": set(), "children": set()},
"Q": {"parents": set(), "children": set()},
}
correct_ancestors2 = [
("A", set()),
("B", set(["A"])),
("C", set(["A", "B"])),
("P", set()),
("Q", set()),
]
# Minimal set for graph 2
correct_minimal_set2 = {
"A": set(["t1", "t2", "t3"]),
"B": set(["t4"]),
"C": set(["t5"]),
"P": set(["t1"]),
"Q": set(["t1", "t7"]),
}
class TestSuiteHierarchy(unittest.TestCase):
def test_compute_dag_empty(self):
correct_dag = {}
self.assertEqual(correct_dag, compute_dag({}))
def test_compute_dag_graph1(self):
for graph in graph1:
dag = compute_dag(graph)
self.assertEqual(
correct_dag1,
dag,
f"Expected: \n{correct_dag1}\nbut received\n{dag}.\nTest case:\n{graph}",
)
def test_compute_dag_graph2(self):
for graph in graph2:
dag = compute_dag(graph)
self.assertEqual(
correct_dag2,
dag,
f"Expected: \n{correct_dag2}\nbut received\n{dag}.\nTest case:\n{graph}",
)
def test_compute_ancestors_graph1(self):
for node, answer in correct_ancestors1:
ancestors = compute_ancestors(node, correct_dag1)
self.assertEqual(answer, ancestors, f"Expected\n{answer}\nbut received\n{ancestors}")
def test_compute_ancestors_graph2(self):
for node, answer in correct_ancestors2:
ancestors = compute_ancestors(node, correct_dag2)
self.assertEqual(answer, ancestors, f"Expected\n{answer}\nbut received\n{ancestors}")
def test_compute_minimal_test_set1(self):
for node, tests in correct_minimal_set1.items():
computed_tests = compute_minimal_test_set(node, correct_dag1, tests_in_suite)
self.assertEqual(
tests,
computed_tests,
f"On node {node}\nExpected\n{tests}\nbut received\n{computed_tests}",
)
def test_compute_minimal_test_set2(self):
for node, tests in correct_minimal_set2.items():
computed_tests = compute_minimal_test_set(node, correct_dag2, tests_in_suite)
self.assertEqual(
tests,
computed_tests,
f"On node {node}\nExpected\n{tests}\nbut received\n{computed_tests}",
)
def test_union_of_minimal_equals_union_of_full(self):
# Test that the union(minimal set of each ancestor) is equal
# to the union(full test set of each ancestor)
for node, ancestors in correct_ancestors1:
min_set_union = set()
full_union = set()
for ancestor in ancestors:
min_set_union = min_set_union.union(
compute_minimal_test_set(ancestor, correct_dag1, tests_in_suite)
)
full_union = full_union.union(tests_in_suite[ancestor])
self.assertEqual(
min_set_union,
full_union,
f"Min set union\n{min_set_union}\n != full union\n{full_union}",
)
for node, ancestors in correct_ancestors2:
min_set_union = set()
full_union = set()
for ancestor in ancestors:
min_set_union = min_set_union.union(
compute_minimal_test_set(ancestor, correct_dag2, tests_in_suite)
)
full_union = full_union.union(tests_in_suite[ancestor])
self.assertEqual(
min_set_union,
full_union,
f"Min set union\n{min_set_union}\n != full union\n{full_union}",
)

2
evergreen/prelude_mongo_task_generator.sh
View File

@ -1,6 +1,6 @@
function setup_mongo_task_generator {
if [ ! -f mongo-task-generator ]; then
curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.7.17/mongo-task-generator --output mongo-task-generator
curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.7.18/mongo-task-generator --output mongo-task-generator
chmod +x mongo-task-generator
fi
}