mirror of https://github.com/mongodb/mongo
356 lines
14 KiB
Python
356 lines
14 KiB
Python
"""Unit tests for suite_split.py."""
|
|
import unittest
|
|
from datetime import datetime
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import requests
|
|
|
|
import buildscripts.task_generation.suite_split as under_test
|
|
from buildscripts.task_generation.suite_split_strategies import greedy_division, \
|
|
round_robin_fallback
|
|
from buildscripts.util.teststats import TestRuntime, HistoricalTestInformation
|
|
|
|
# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access
|
|
|
|
|
|
def mock_evg_error(mock_evg_api, error_code=requests.codes.SERVICE_UNAVAILABLE):
|
|
response = MagicMock(status_code=error_code)
|
|
mock_evg_api.test_stats_by_project.side_effect = requests.HTTPError(response=response)
|
|
return mock_evg_api
|
|
|
|
|
|
def build_mock_service(evg_api=None, split_config=None, resmoke_proxy=None):
|
|
|
|
return under_test.SuiteSplitService(
|
|
evg_api=evg_api if evg_api else MagicMock(),
|
|
resmoke_proxy=resmoke_proxy if resmoke_proxy else MagicMock(),
|
|
config=split_config if split_config else MagicMock(),
|
|
split_strategy=greedy_division,
|
|
fallback_strategy=round_robin_fallback,
|
|
)
|
|
|
|
|
|
def tst_stat_mock(file, duration, pass_count):
|
|
return HistoricalTestInformation(
|
|
test_name=file,
|
|
num_pass=pass_count,
|
|
num_fail=0,
|
|
avg_duration_pass=duration,
|
|
)
|
|
|
|
|
|
def build_mock_split_config(target_resmoke_time=None, max_sub_suites=None):
|
|
return under_test.SuiteSplitConfig(
|
|
evg_project="project",
|
|
target_resmoke_time=target_resmoke_time if target_resmoke_time else 60,
|
|
max_sub_suites=max_sub_suites if max_sub_suites else 1000,
|
|
max_tests_per_suite=100,
|
|
start_date=datetime.utcnow(),
|
|
end_date=datetime.utcnow(),
|
|
)
|
|
|
|
|
|
def build_mock_split_params(test_filter=None):
|
|
return under_test.SuiteSplitParameters(
|
|
build_variant="build variant",
|
|
task_name="task name",
|
|
suite_name="suite name",
|
|
filename="targetfile",
|
|
test_file_filter=test_filter,
|
|
)
|
|
|
|
|
|
def build_mock_sub_suite(index, test_list):
|
|
return under_test.SubSuite(
|
|
index=index,
|
|
suite_name="suite_name",
|
|
test_list=test_list,
|
|
tests_with_runtime_info=0,
|
|
max_test_runtime=0,
|
|
historic_runtime=0,
|
|
task_overhead=0,
|
|
)
|
|
|
|
|
|
class TestSubSuite(unittest.TestCase):
|
|
def test_tests_with_0_runtime_should_not_override_timeouts(self):
|
|
test_list = [f"test_{i}" for i in range(10)]
|
|
runtime_list = [
|
|
MagicMock(spec_set=TestRuntime, test_name=test, runtime=3.14) for test in test_list
|
|
]
|
|
runtime_list[3].runtime = 0
|
|
sub_suite = under_test.SubSuite.from_test_list(0, "my_suite", test_list, None, runtime_list)
|
|
|
|
assert not sub_suite.should_overwrite_timeout()
|
|
|
|
def test_tests_with_full_runtime_history_should_override_timeouts(self):
|
|
test_list = [f"test_{i}" for i in range(10)]
|
|
runtime_list = [
|
|
MagicMock(spec_set=TestRuntime, test_name=test, runtime=3.14) for test in test_list
|
|
]
|
|
sub_suite = under_test.SubSuite.from_test_list(0, "my_suite", test_list, None, runtime_list)
|
|
|
|
assert sub_suite.should_overwrite_timeout()
|
|
|
|
|
|
class TestGeneratedSuite(unittest.TestCase):
|
|
def test_get_test_list_should_run_tests_in_sub_tasks(self):
|
|
n_sub_suites = 3
|
|
n_tests_per_suite = 5
|
|
test_lists = [[f"test_{i * n_tests_per_suite + j}" for j in range(n_tests_per_suite)]
|
|
for i in range(n_sub_suites)]
|
|
mock_sub_suites = [build_mock_sub_suite(i, test_lists[i]) for i in range(n_sub_suites)]
|
|
mock_suite = under_test.GeneratedSuite(sub_suites=mock_sub_suites,
|
|
build_variant="build_variant", task_name="task_name",
|
|
suite_name="suite_name", filename="filename")
|
|
|
|
all_tests = mock_suite.get_test_list()
|
|
|
|
for test_list in test_lists:
|
|
for test in test_list:
|
|
self.assertIn(test, all_tests)
|
|
|
|
def test_sub_suite_config_file_should_generate_filename_for_sub_suites(self):
|
|
task_name = "task_name"
|
|
n_sub_suites = 42
|
|
mock_sub_suites = [build_mock_sub_suite(i, []) for i in range(n_sub_suites)]
|
|
mock_suite = under_test.GeneratedSuite(
|
|
sub_suites=mock_sub_suites, build_variant="build_variant", task_name=f"{task_name}_gen",
|
|
suite_name="suite_name", filename="filename")
|
|
|
|
self.assertEqual(mock_suite.sub_suite_config_file(34), "task_name_34")
|
|
self.assertEqual(mock_suite.sub_suite_config_file(0), "task_name_00")
|
|
self.assertEqual(mock_suite.sub_suite_config_file(3), "task_name_03")
|
|
self.assertEqual(mock_suite.sub_suite_config_file(None), "task_name_misc")
|
|
|
|
|
|
class TestSplitSuite(unittest.TestCase):
|
|
@patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3")
|
|
def test_calculate_suites(self, get_stats_from_s3_mock):
|
|
mock_test_stats = [tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)]
|
|
split_config = build_mock_split_config(target_resmoke_time=10)
|
|
split_params = build_mock_split_params()
|
|
|
|
suite_split_service = build_mock_service(split_config=split_config)
|
|
get_stats_from_s3_mock.return_value = mock_test_stats
|
|
suite_split_service.resmoke_proxy.list_tests.return_value = [
|
|
stat.test_name for stat in mock_test_stats
|
|
]
|
|
suite_split_service.resmoke_proxy.read_suite_config.return_value = {}
|
|
|
|
with patch("os.path.exists") as exists_mock:
|
|
exists_mock.return_value = True
|
|
|
|
suite = suite_split_service.split_suite(split_params)
|
|
|
|
# There are 100 tests taking 1 minute, with a target of 10 min we expect 10 suites.
|
|
self.assertEqual(10, len(suite))
|
|
for sub_suite in suite.sub_suites:
|
|
self.assertEqual(10, len(sub_suite.test_list))
|
|
|
|
@patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3")
|
|
def test_calculate_suites_uses_fallback_on_no_results(self, get_stats_from_s3_mock):
|
|
n_tests = 100
|
|
max_sub_suites = 5
|
|
split_config = build_mock_split_config(max_sub_suites=max_sub_suites)
|
|
split_params = build_mock_split_params()
|
|
|
|
suite_split_service = build_mock_service(split_config=split_config)
|
|
get_stats_from_s3_mock.return_value = []
|
|
suite_split_service.resmoke_proxy.list_tests.return_value = [
|
|
f"test_{i}.js" for i in range(n_tests)
|
|
]
|
|
|
|
suite = suite_split_service.split_suite(split_params)
|
|
|
|
self.assertEqual(max_sub_suites, len(suite))
|
|
for sub_suite in suite.sub_suites:
|
|
self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list))
|
|
|
|
@patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3")
|
|
def test_calculate_suites_uses_fallback_if_only_results_are_filtered(
|
|
self, get_stats_from_s3_mock):
|
|
n_tests = 100
|
|
max_sub_suites = 10
|
|
mock_test_stats = [tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)]
|
|
split_config = build_mock_split_config(target_resmoke_time=10,
|
|
max_sub_suites=max_sub_suites)
|
|
split_params = build_mock_split_params()
|
|
|
|
suite_split_service = build_mock_service(split_config=split_config)
|
|
get_stats_from_s3_mock.return_value = mock_test_stats
|
|
suite_split_service.resmoke_proxy.list_tests.return_value = [
|
|
f"test_{i}.js" for i in range(n_tests)
|
|
]
|
|
suite_split_service.resmoke_proxy.read_suite_config.return_value = {}
|
|
|
|
with patch("os.path.exists") as exists_mock:
|
|
exists_mock.return_value = False
|
|
|
|
suite = suite_split_service.split_suite(split_params)
|
|
|
|
# There are 100 tests taking 1 minute, with a target of 10 min we expect 10 suites.
|
|
self.assertEqual(max_sub_suites, len(suite))
|
|
for sub_suite in suite.sub_suites:
|
|
self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list))
|
|
|
|
@patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3")
|
|
def test_calculate_suites_will_filter_specified_tests(self, get_stats_from_s3_mock):
|
|
mock_test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(100)]
|
|
split_config = build_mock_split_config(target_resmoke_time=10)
|
|
split_params = build_mock_split_params(
|
|
test_filter=lambda t: t in {"test_1.js", "test_2.js"})
|
|
|
|
suite_split_service = build_mock_service(split_config=split_config)
|
|
get_stats_from_s3_mock.return_value = mock_test_stats
|
|
suite_split_service.resmoke_proxy.list_tests.return_value = [
|
|
stat.test_name for stat in mock_test_stats
|
|
]
|
|
suite_split_service.resmoke_proxy.read_suite_config.return_value = {}
|
|
|
|
with patch("os.path.exists") as exists_mock:
|
|
exists_mock.return_value = True
|
|
|
|
suite = suite_split_service.split_suite(split_params)
|
|
|
|
self.assertEqual(1, len(suite))
|
|
for sub_suite in suite.sub_suites:
|
|
self.assertEqual(2, len(sub_suite.test_list))
|
|
self.assertIn("test_1.js", sub_suite.test_list)
|
|
self.assertIn("test_2.js", sub_suite.test_list)
|
|
|
|
|
|
class TestFilterTests(unittest.TestCase):
|
|
def test_filter_missing_files(self):
|
|
tests_runtimes = [
|
|
TestRuntime(test_name="dir1/file1.js", runtime=20.32),
|
|
TestRuntime(test_name="dir2/file2.js", runtime=24.32),
|
|
TestRuntime(test_name="dir1/file3.js", runtime=36.32),
|
|
]
|
|
mock_params = MagicMock(test_file_filter=None)
|
|
mock_resmoke_proxy = MagicMock()
|
|
mock_resmoke_proxy.list_tests.return_value = [
|
|
runtime.test_name for runtime in tests_runtimes
|
|
]
|
|
suite_split_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)
|
|
|
|
with patch("os.path.exists") as exists_mock:
|
|
exists_mock.side_effect = [False, True, True]
|
|
filtered_list = suite_split_service.filter_tests(tests_runtimes, mock_params)
|
|
|
|
self.assertEqual(2, len(filtered_list))
|
|
self.assertNotIn(tests_runtimes[0], filtered_list)
|
|
self.assertIn(tests_runtimes[2], filtered_list)
|
|
self.assertIn(tests_runtimes[1], filtered_list)
|
|
|
|
def test_filter_blacklist_files(self):
|
|
tests_runtimes = [
|
|
TestRuntime(test_name="dir1/file1.js", runtime=20.32),
|
|
TestRuntime(test_name="dir2/file2.js", runtime=24.32),
|
|
TestRuntime(test_name="dir1/file3.js", runtime=36.32),
|
|
]
|
|
blacklisted_test = tests_runtimes[1][0]
|
|
mock_params = MagicMock(test_file_filter=None)
|
|
mock_resmoke_proxy = MagicMock()
|
|
mock_resmoke_proxy.list_tests.return_value = [
|
|
runtime.test_name for runtime in tests_runtimes if runtime.test_name != blacklisted_test
|
|
]
|
|
suite_split_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)
|
|
|
|
with patch("os.path.exists") as exists_mock:
|
|
exists_mock.return_value = True
|
|
|
|
filtered_list = suite_split_service.filter_tests(tests_runtimes, mock_params)
|
|
|
|
self.assertEqual(2, len(filtered_list))
|
|
self.assertNotIn(blacklisted_test, filtered_list)
|
|
self.assertIn(tests_runtimes[2], filtered_list)
|
|
self.assertIn(tests_runtimes[0], filtered_list)
|
|
|
|
def test_filter_blacklist_files_for_windows(self):
|
|
tests_runtimes = [
|
|
TestRuntime(test_name="dir1/file1.js", runtime=20.32),
|
|
TestRuntime(test_name="dir2/file2.js", runtime=24.32),
|
|
TestRuntime(test_name="dir1/dir3/file3.js", runtime=36.32),
|
|
]
|
|
|
|
blacklisted_test = tests_runtimes[1][0]
|
|
|
|
mock_params = MagicMock(test_file_filter=None)
|
|
mock_resmoke_proxy = MagicMock()
|
|
mock_resmoke_proxy.list_tests.return_value = [
|
|
runtime.test_name.replace("/", "\\") for runtime in tests_runtimes
|
|
if runtime.test_name != blacklisted_test
|
|
]
|
|
suite_split_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)
|
|
|
|
with patch("os.path.exists") as exists_mock:
|
|
exists_mock.return_value = True
|
|
|
|
filtered_list = suite_split_service.filter_tests(tests_runtimes, mock_params)
|
|
|
|
self.assertNotIn(blacklisted_test, filtered_list)
|
|
self.assertIn(tests_runtimes[2], filtered_list)
|
|
self.assertIn(tests_runtimes[0], filtered_list)
|
|
self.assertEqual(2, len(filtered_list))
|
|
|
|
|
|
class TestGetCleanEveryNCadence(unittest.TestCase):
|
|
def test_clean_every_n_cadence_on_asan(self):
|
|
split_config = MagicMock(
|
|
san_options="ASAN_OPTIONS=\"detect_leaks=1:check_initialization_order=true\"")
|
|
suite_split_service = build_mock_service(split_config=split_config)
|
|
|
|
cadence = suite_split_service._get_clean_every_n_cadence("suite", True)
|
|
|
|
self.assertEqual(1, cadence)
|
|
|
|
def test_clean_every_n_cadence_from_hook_config(self):
|
|
expected_n = 42
|
|
mock_resmoke_proxy = MagicMock()
|
|
mock_resmoke_proxy.read_suite_config.return_value = {
|
|
"executor": {
|
|
"hooks": [{
|
|
"class": "hook1",
|
|
}, {
|
|
"class": under_test.CLEAN_EVERY_N_HOOK,
|
|
"n": expected_n,
|
|
}]
|
|
}
|
|
}
|
|
suite_split_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)
|
|
|
|
cadence = suite_split_service._get_clean_every_n_cadence("suite", False)
|
|
|
|
self.assertEqual(expected_n, cadence)
|
|
|
|
def test_clean_every_n_cadence_no_n_in_hook_config(self):
|
|
mock_resmoke_proxy = MagicMock()
|
|
mock_resmoke_proxy.read_suite_config.return_value = {
|
|
"executor": {
|
|
"hooks": [{
|
|
"class": "hook1",
|
|
}, {
|
|
"class": under_test.CLEAN_EVERY_N_HOOK,
|
|
}]
|
|
}
|
|
}
|
|
suite_split_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)
|
|
|
|
cadence = suite_split_service._get_clean_every_n_cadence("suite", False)
|
|
|
|
self.assertEqual(1, cadence)
|
|
|
|
def test_clean_every_n_cadence_no_hook_config(self):
|
|
mock_resmoke_proxy = MagicMock()
|
|
mock_resmoke_proxy.read_suite_config.return_value = {
|
|
"executor": {"hooks": [{
|
|
"class": "hook1",
|
|
}, ]}
|
|
}
|
|
suite_split_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)
|
|
|
|
cadence = suite_split_service._get_clean_every_n_cadence("suite", False)
|
|
|
|
self.assertEqual(1, cadence)
|