"""Unit tests for buildscripts/burn_in_tests_multiversion.py.""" from __future__ import absolute_import import json from datetime import datetime import os import sys import unittest from mock import MagicMock, patch import inject from evergreen import EvergreenApi import buildscripts.burn_in_tests_multiversion as under_test from buildscripts.burn_in_tests import TaskInfo from buildscripts.ciconfig.evergreen import parse_evergreen_file, EvergreenProjectConfig import buildscripts.resmokelib.parser as _parser from buildscripts.evergreen_burn_in_tests import EvergreenFileChangeDetector from buildscripts.task_generation.gen_config import GenerationConfiguration from buildscripts.task_generation.multiversion_util import REPL_MIXED_VERSION_CONFIGS, \ SHARDED_MIXED_VERSION_CONFIGS from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig from buildscripts.task_generation.suite_split import SuiteSplitConfig from buildscripts.task_generation.suite_split_strategies import greedy_division, SplitStrategy, \ FallbackStrategy, round_robin_fallback from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions _parser.set_run_options() MONGO_4_2_HASH = "d94888c0d0a8065ca57d354ece33b3c2a1a5a6d6" # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access,no-value-for-parameter def create_tests_by_task_mock(n_tasks, n_tests, multiversion_values=None): if multiversion_values is None: multiversion_values = [None for _ in range(n_tasks)] return { f"task_{i}_gen": TaskInfo( display_task_name=f"task_{i}", resmoke_args=f"--suites=suite_{i}", tests=[f"jstests/tests_{j}" for j in range(n_tests)], require_multiversion=multiversion_values[i], distro="", ) for i in range(n_tasks) } MV_MOCK_SUITES = ["replica_sets_jscore_passthrough", "sharding_jscore_passthrough"] MV_MOCK_TESTS = { "replica_sets_jscore_passthrough": [ "core/all.js", "core/andor.js", "core/apitest_db.js", "core/auth1.js", "core/auth2.js", ], "sharding_jscore_passthrough": [ "core/basic8.js", "core/batch_size.js", "core/bson.js", "core/bulk_insert.js", "core/capped.js", ] } def create_multiversion_tests_by_task_mock(n_tasks, n_tests): assert n_tasks <= len(MV_MOCK_SUITES) assert n_tests <= len(MV_MOCK_TESTS[MV_MOCK_SUITES[0]]) return { f"{MV_MOCK_SUITES[i]}": TaskInfo( display_task_name=f"task_{i}", resmoke_args=f"--suites=suite_{i}", tests=[f"jstests/{MV_MOCK_TESTS[MV_MOCK_SUITES[i]][j]}" for j in range(n_tests)], require_multiversion=None, distro="", ) for i in range(n_tasks) } _DATE = datetime(2018, 7, 15) BURN_IN_TESTS = "buildscripts.burn_in_tests" NUM_REPL_MIXED_VERSION_CONFIGS = len(REPL_MIXED_VERSION_CONFIGS) NUM_SHARDED_MIXED_VERSION_CONFIGS = len(SHARDED_MIXED_VERSION_CONFIGS) NS = "buildscripts.burn_in_tests_multiversion" def ns(relative_name): # pylint: disable=invalid-name """Return a full name from a name relative to the test module"s name space.""" return NS + "." + relative_name def bit_ns(relative_name): return BURN_IN_TESTS + "." + relative_name def mock_a_file(filename): change = MagicMock(a_path=filename) return change def mock_git_diff(change_list): diff = MagicMock() diff.iter_change_type.return_value = change_list return diff def mock_changed_git_files(add_files): repo = MagicMock() repo.index.diff.return_value = mock_git_diff([mock_a_file(f) for f in add_files]) return repo def get_evergreen_config(config_file_path): evergreen_home = os.path.expanduser(os.path.join("~", "evergreen")) if os.path.exists(evergreen_home): return parse_evergreen_file(config_file_path, evergreen_home) return parse_evergreen_file(config_file_path) def create_variant_task_mock(task_name, suite_name, distro="distro"): variant_task = MagicMock() variant_task.name = task_name variant_task.generated_task_name = task_name variant_task.resmoke_suite = suite_name variant_task.get_vars_suite_name.return_value = suite_name variant_task.combined_resmoke_args = f"--suites={suite_name}" variant_task.require_multiversion = None variant_task.run_on = [distro] return variant_task def build_mock_gen_task_options(): return GenTaskOptions( create_misc_suite=False, is_patch=True, generated_config_dir=under_test.DEFAULT_CONFIG_DIR, use_default_timeouts=False, ) def build_mock_split_task_config(): return SuiteSplitConfig( evg_project="my project", target_resmoke_time=60, max_sub_suites=100, max_tests_per_suite=1, start_date=datetime.utcnow(), end_date=datetime.utcnow(), default_to_fallback=True, ) def configure_dependencies(evg_api, split_config): gen_task_options = build_mock_gen_task_options() def dependencies(binder: inject.Binder) -> None: binder.bind(SuiteSplitConfig, split_config) binder.bind(SplitStrategy, greedy_division) binder.bind(FallbackStrategy, round_robin_fallback) binder.bind(GenTaskOptions, gen_task_options) binder.bind(EvergreenApi, evg_api) binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file()) binder.bind(ResmokeProxyConfig, ResmokeProxyConfig(resmoke_suite_dir=under_test.DEFAULT_TEST_SUITE_DIR)) binder.bind(EvergreenFileChangeDetector, None) binder.bind(EvergreenProjectConfig, MagicMock()) binder.bind( under_test.BurnInConfig, under_test.BurnInConfig(build_id="build_id", build_variant="build variant", revision="revision")) inject.clear_and_configure(dependencies) class TestCreateMultiversionGenerateTasksConfig(unittest.TestCase): def tests_no_tasks_given(self): target_file = "target_file.json" mock_evg_api = MagicMock() split_config = build_mock_split_task_config() configure_dependencies(mock_evg_api, split_config) orchestrator = under_test.MultiversionBurnInOrchestrator() generated_config = orchestrator.generate_configuration({}, target_file, "build_variant") evg_config = [ config for config in generated_config.file_list if config.file_name == target_file ] self.assertEqual(1, len(evg_config)) evg_config = evg_config[0] evg_config_dict = json.loads(evg_config.content) self.assertEqual(0, len(evg_config_dict["tasks"])) def test_tasks_not_in_multiversion_suites(self): n_tasks = 1 n_tests = 1 target_file = "target_file.json" mock_evg_api = MagicMock() split_config = build_mock_split_task_config() configure_dependencies(mock_evg_api, split_config) tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) orchestrator = under_test.MultiversionBurnInOrchestrator() generated_config = orchestrator.generate_configuration(tests_by_task, target_file, "build_variant") evg_config = [ config for config in generated_config.file_list if config.file_name == target_file ] self.assertEqual(1, len(evg_config)) evg_config = evg_config[0] evg_config_dict = json.loads(evg_config.content) self.assertEqual(0, len(evg_config_dict["tasks"])) @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") @patch( "buildscripts.evergreen_gen_multiversion_tests.get_backports_required_hash_for_shell_version" ) def test_one_task_one_test(self, mock_hash): mock_hash.return_value = MONGO_4_2_HASH n_tasks = 1 n_tests = 1 target_file = "target_file.json" mock_evg_api = MagicMock() split_config = build_mock_split_task_config() configure_dependencies(mock_evg_api, split_config) tests_by_task = create_multiversion_tests_by_task_mock(n_tasks, n_tests) orchestrator = under_test.MultiversionBurnInOrchestrator() generated_config = orchestrator.generate_configuration(tests_by_task, target_file, "build_variant") evg_config = [ config for config in generated_config.file_list if config.file_name == target_file ] self.assertEqual(1, len(evg_config)) evg_config = evg_config[0] evg_config_dict = json.loads(evg_config.content) tasks = evg_config_dict["tasks"] self.assertEqual(len(tasks), NUM_REPL_MIXED_VERSION_CONFIGS * n_tests) @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") @patch( "buildscripts.evergreen_gen_multiversion_tests.get_backports_required_hash_for_shell_version" ) def test_n_task_one_test(self, mock_hash): mock_hash.return_value = MONGO_4_2_HASH n_tasks = 2 n_tests = 1 target_file = "target_file.json" mock_evg_api = MagicMock() split_config = build_mock_split_task_config() configure_dependencies(mock_evg_api, split_config) tests_by_task = create_multiversion_tests_by_task_mock(n_tasks, n_tests) orchestrator = under_test.MultiversionBurnInOrchestrator() generated_config = orchestrator.generate_configuration(tests_by_task, target_file, "build_variant") evg_config = [ config for config in generated_config.file_list if config.file_name == target_file ] self.assertEqual(1, len(evg_config)) evg_config = evg_config[0] evg_config_dict = json.loads(evg_config.content) tasks = evg_config_dict["tasks"] self.assertEqual( len(tasks), (NUM_REPL_MIXED_VERSION_CONFIGS + NUM_SHARDED_MIXED_VERSION_CONFIGS) * n_tests) @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") @patch( "buildscripts.evergreen_gen_multiversion_tests.get_backports_required_hash_for_shell_version" ) def test_one_task_n_test(self, mock_hash): mock_hash.return_value = MONGO_4_2_HASH n_tasks = 1 n_tests = 2 target_file = "target_file.json" mock_evg_api = MagicMock() split_config = build_mock_split_task_config() configure_dependencies(mock_evg_api, split_config) tests_by_task = create_multiversion_tests_by_task_mock(n_tasks, n_tests) orchestrator = under_test.MultiversionBurnInOrchestrator() generated_config = orchestrator.generate_configuration(tests_by_task, target_file, "build_variant") evg_config = [ config for config in generated_config.file_list if config.file_name == target_file ] self.assertEqual(1, len(evg_config)) evg_config = evg_config[0] evg_config_dict = json.loads(evg_config.content) tasks = evg_config_dict["tasks"] self.assertEqual(len(tasks), NUM_REPL_MIXED_VERSION_CONFIGS * n_tests) @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") @patch( "buildscripts.evergreen_gen_multiversion_tests.get_backports_required_hash_for_shell_version" ) def test_n_task_m_test(self, mock_hash): mock_hash.return_value = MONGO_4_2_HASH n_tasks = 2 n_tests = 3 target_file = "target_file.json" mock_evg_api = MagicMock() split_config = build_mock_split_task_config() configure_dependencies(mock_evg_api, split_config) tests_by_task = create_multiversion_tests_by_task_mock(n_tasks, n_tests) orchestrator = under_test.MultiversionBurnInOrchestrator() generated_config = orchestrator.generate_configuration(tests_by_task, target_file, "build_variant") evg_config = [ config for config in generated_config.file_list if config.file_name == target_file ] self.assertEqual(1, len(evg_config)) evg_config = evg_config[0] evg_config_dict = json.loads(evg_config.content) tasks = evg_config_dict["tasks"] self.assertEqual( len(tasks), (NUM_REPL_MIXED_VERSION_CONFIGS + NUM_SHARDED_MIXED_VERSION_CONFIGS) * n_tests) class TestGenerateConfig(unittest.TestCase): def test_validate_multiversion(self): evg_conf_mock = MagicMock() gen_config = under_test.GenerateConfig("build_variant", "project") gen_config.validate(evg_conf_mock) class TestGatherTaskInfo(unittest.TestCase): def test_multiversion_task(self): suite_name = "suite_1" distro_name = "distro_1" variant = "build_variant" evg_conf_mock = MagicMock() evg_conf_mock.get_task.return_value.is_generate_resmoke_task = False task_mock = create_variant_task_mock("task 1", suite_name, distro_name) task_mock.require_multiversion = True test_list = [f"test{i}.js" for i in range(3)] tests_by_suite = { suite_name: test_list, "suite 2": [f"test{i}.js" for i in range(1)], "suite 3": [f"test{i}.js" for i in range(2)], } task_info = TaskInfo.from_task(task_mock, tests_by_suite, evg_conf_mock, variant) self.assertIn(suite_name, task_info.resmoke_args) for test in test_list: self.assertIn(test, task_info.tests) self.assertEqual(task_mock.require_multiversion, task_info.require_multiversion) self.assertEqual(distro_name, task_info.distro)