"""Unit tests for the selected_tests script.""" import json import sys import unittest from datetime import datetime, timedelta from typing import Dict, Any import inject from mock import MagicMock, patch from evergreen import EvergreenApi # pylint: disable=wrong-import-position import buildscripts.ciconfig.evergreen as _evergreen from buildscripts.burn_in_tests import TaskInfo from buildscripts.patch_builds.selected_tests.selected_tests_client import SelectedTestsClient, \ TestMappingsResponse, TestMapping, TestFileInstance, TaskMappingsResponse, TaskMapInstance, \ TaskMapping from buildscripts.selected_tests import EvgExpansions from buildscripts.task_generation.gen_config import GenerationConfiguration from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig from buildscripts.task_generation.suite_split import SuiteSplitConfig from buildscripts.task_generation.suite_split_strategies import SplitStrategy, greedy_division, \ FallbackStrategy, round_robin_fallback from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions from buildscripts.tests.test_burn_in_tests import get_evergreen_config, mock_changed_git_files from buildscripts import selected_tests as under_test # pylint: disable=missing-docstring,invalid-name,unused-argument,protected-access,no-value-for-parameter,too-many-locals NS = "buildscripts.selected_tests" def ns(relative_name): # pylint: disable=invalid-name """Return a full name from a name relative to the test module"s name space.""" return NS + "." + relative_name def empty_build_variant(variant_name: str) -> Dict[str, Any]: return { "buildvariants": [{ "name": variant_name, "tasks": [], }], "tasks": [], } def configure_dependencies(evg_api, evg_expansions, evg_project_config, selected_test_client, test_suites_dir=under_test.DEFAULT_TEST_SUITE_DIR): start_date = datetime.utcnow() end_date = start_date - timedelta(weeks=2) def dependencies(binder: inject.Binder) -> None: binder.bind(EvgExpansions, evg_expansions) binder.bind(_evergreen.EvergreenProjectConfig, evg_project_config) binder.bind(SuiteSplitConfig, evg_expansions.build_suite_split_config(start_date, end_date)) binder.bind(SplitStrategy, greedy_division) binder.bind(FallbackStrategy, round_robin_fallback) binder.bind(GenTaskOptions, evg_expansions.build_gen_task_options()) binder.bind(EvergreenApi, evg_api) binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file()) binder.bind(ResmokeProxyConfig, ResmokeProxyConfig(resmoke_suite_dir=test_suites_dir)) binder.bind(SelectedTestsClient, selected_test_client) inject.clear_and_configure(dependencies) class TestAcceptance(unittest.TestCase): """A suite of Acceptance tests for selected_tests.""" @staticmethod def _mock_evg_api(): evg_api_mock = MagicMock() task_mock = evg_api_mock.task_by_id.return_value task_mock.execution = 0 return evg_api_mock @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") def test_when_no_mappings_are_found_for_changed_files(self): mock_evg_api = self._mock_evg_api() mock_evg_config = get_evergreen_config("etc/evergreen.yml") mock_evg_expansions = under_test.EvgExpansions( task_id="task_id", task_name="selected_tests_gen", build_variant="selected-tests", build_id="my_build_id", project="mongodb-mongo-master", revision="abc123", version_id="my_version", ) mock_selected_tests_client = MagicMock() mock_selected_tests_client.get_test_mappings.return_value = TestMappingsResponse( test_mappings=[]) configure_dependencies(mock_evg_api, mock_evg_expansions, mock_evg_config, mock_selected_tests_client) repos = [mock_changed_git_files([])] selected_tests = under_test.SelectedTestsOrchestrator() changed_files = selected_tests.find_changed_files(repos, "task_id") generated_config = selected_tests.generate_version(changed_files) # assert that config_dict does not contain keys for any generated task configs self.assertEqual(len(generated_config.file_list), 1) self.assertEqual(generated_config.file_list[0].file_name, "selected_tests_config.json") @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") def test_when_test_mappings_are_found_for_changed_files(self, get_stats_from_s3_mock): get_stats_from_s3_mock.return_value = [] mock_evg_api = self._mock_evg_api() mock_evg_config = get_evergreen_config("etc/evergreen.yml") mock_evg_expansions = under_test.EvgExpansions( task_id="task_id", task_name="selected_tests_gen", build_variant="selected-tests", build_id="my_build_id", project="mongodb-mongo-master", revision="abc123", version_id="my_version", ) mock_test_mapping = TestMapping( branch="master", project="mongodb-mongo-master", repo="mongodb/mongo", source_file="src/file1.cpp", source_file_seen_count=8, test_files=[TestFileInstance(name="jstests/auth/auth1.js", test_file_seen_count=3)]) mock_selected_tests_client = MagicMock() mock_selected_tests_client.get_test_mappings.return_value = TestMappingsResponse( test_mappings=[mock_test_mapping]) configure_dependencies(mock_evg_api, mock_evg_expansions, mock_evg_config, mock_selected_tests_client) repos = [mock_changed_git_files(["src/file1.cpp"])] selected_tests = under_test.SelectedTestsOrchestrator() changed_files = selected_tests.find_changed_files(repos, "task_id") generated_config = selected_tests.generate_version(changed_files) files_to_generate = {gen_file.file_name for gen_file in generated_config.file_list} self.assertIn("selected_tests_config.json", files_to_generate) # assert that generated suite files have the suite name and the variant name in the # filename, to prevent tasks on different variants from using the same suite file self.assertIn("auth_enterprise-rhel-8-64-bit-dynamic-required_0.yml", files_to_generate) generated_evg_config_raw = [ gen_file.content for gen_file in generated_config.file_list if gen_file.file_name == "selected_tests_config.json" ][0] generated_evg_config = json.loads(generated_evg_config_raw) build_variants_with_generated_tasks = generated_evg_config["buildvariants"] # jstests/auth/auth1.js belongs to two suites, auth and auth_audit, rhel_80_with_generated_tasks = next( (variant for variant in build_variants_with_generated_tasks if variant["name"] == "enterprise-rhel-8-64-bit-dynamic-required"), None) self.assertEqual(len(rhel_80_with_generated_tasks["tasks"]), 2) @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") def test_when_task_mappings_are_found_for_changed_files(self, get_stats_from_s3_mock): get_stats_from_s3_mock.return_value = [] mock_evg_api = self._mock_evg_api() mock_evg_config = get_evergreen_config("etc/evergreen.yml") mock_evg_expansions = under_test.EvgExpansions( task_id="task_id", task_name="selected_tests_gen", build_variant="selected-tests", build_id="my_build_id", project="mongodb-mongo-master", revision="abc123", version_id="my_version", ) mock_task_mapping = TaskMapping( branch="master", project="mongodb-mongo-master", repo="mongodb/mongo", source_file="src/file1.cpp", source_file_seen_count=8, tasks=[TaskMapInstance(name="auth", variant="enterprise-rhel-80", flip_count=5)]) mock_selected_tests_client = MagicMock() mock_selected_tests_client.get_task_mappings.return_value = TaskMappingsResponse( task_mappings=[mock_task_mapping]) configure_dependencies(mock_evg_api, mock_evg_expansions, mock_evg_config, mock_selected_tests_client) repos = [mock_changed_git_files(["src/file1.cpp"])] selected_tests = under_test.SelectedTestsOrchestrator() changed_files = selected_tests.find_changed_files(repos, "task_id") generated_config = selected_tests.generate_version(changed_files) files_to_generate = {gen_file.file_name for gen_file in generated_config.file_list} self.assertIn("selected_tests_config.json", files_to_generate) generated_evg_config_raw = [ gen_file.content for gen_file in generated_config.file_list if gen_file.file_name == "selected_tests_config.json" ][0] generated_evg_config = json.loads(generated_evg_config_raw) # the auth task's generator task, max_sub_suites is 3, # resulting in 3 subtasks being generated, plus a _misc task, hence 4 # tasks total build_variants_with_generated_tasks = generated_evg_config["buildvariants"] rhel_80_with_generated_tasks = next( (variant for variant in build_variants_with_generated_tasks if variant["name"] == "enterprise-rhel-8-64-bit-dynamic-required"), None) self.assertEqual(len(rhel_80_with_generated_tasks["tasks"]), 5) class TestExcludeTask(unittest.TestCase): def test_task_should_not_be_excluded(self): task = _evergreen.Task({"name": "regular_task"}) self.assertEqual(under_test._exclude_task(task), False) def test_task_should_be_excluded(self): excluded_task = under_test.EXCLUDE_TASK_LIST[0] task = _evergreen.Task({"name": excluded_task}) self.assertEqual(under_test._exclude_task(task), True) def test_task_matches_excluded_pattern(self): task_that_matches_exclude_pattern = "compile_all" task = _evergreen.Task({"name": task_that_matches_exclude_pattern}) self.assertEqual(under_test._exclude_task(task), True) def build_mock_evg_task(name, cmd_func="generate resmoke tasks", resmoke_args="--storageEngine=wiredTiger"): return _evergreen.Task({ "name": name, "commands": [{ "func": cmd_func, "vars": {"resmoke_args": resmoke_args, }, }], }) class TestGetEvgTaskConfig(unittest.TestCase): def test_task_is_a_generate_resmoke_task(self): build_variant_conf = MagicMock() build_variant_conf.name = "variant" task = build_mock_evg_task("auth_gen") task_config_service = under_test.TaskConfigService() evg_task_config = task_config_service.get_evg_task_config(task, build_variant_conf) self.assertEqual(evg_task_config["task_name"], "auth") self.assertEqual(evg_task_config["build_variant"], "variant") self.assertIsNone(evg_task_config.get("suite")) self.assertEqual( evg_task_config["resmoke_args"], "--storageEngine=wiredTiger", ) def test_task_is_not_a_generate_resmoke_task(self): build_variant_conf = MagicMock() build_variant_conf.name = "variant" task = build_mock_evg_task("jsCore_auth", "run tests", "--suites=core_auth --storageEngine=wiredTiger") task_config_service = under_test.TaskConfigService() evg_task_config = task_config_service.get_evg_task_config(task, build_variant_conf) self.assertEqual(evg_task_config["task_name"], "jsCore_auth") self.assertEqual(evg_task_config["build_variant"], "variant") self.assertEqual(evg_task_config["suite"], "core_auth") self.assertEqual( evg_task_config["resmoke_args"], "--storageEngine=wiredTiger", ) class TestGetTaskConfigsForTestMappings(unittest.TestCase): @patch(ns("_exclude_task")) @patch(ns("_find_task")) def test_get_config_for_test_mapping(self, find_task_mock, exclude_task_mock): find_task_mock.side_effect = [ build_mock_evg_task("jsCore_auth", "run tests"), build_mock_evg_task("auth_gen", "run tests", "--suites=core_auth --storageEngine=wiredTiger"), ] exclude_task_mock.return_value = False tests_by_task = { "jsCore_auth": TaskInfo( display_task_name="task 1", tests=[ "jstests/core/currentop_waiting_for_latch.js", "jstests/core/latch_analyzer.js", ], resmoke_args="", require_multiversion=None, distro="", ), "auth_gen": TaskInfo( display_task_name="task 2", tests=["jstests/auth/auth3.js"], resmoke_args="", require_multiversion=None, distro="", ), } task_config_service = under_test.TaskConfigService() task_configs = task_config_service.get_task_configs_for_test_mappings( tests_by_task, MagicMock()) self.assertEqual(task_configs["jsCore_auth"]["resmoke_args"], "--storageEngine=wiredTiger") self.assertEqual( task_configs["jsCore_auth"]["selected_tests_to_run"], {"jstests/core/currentop_waiting_for_latch.js", "jstests/core/latch_analyzer.js"}) self.assertEqual(task_configs["auth_gen"]["suite"], "core_auth") self.assertEqual(task_configs["auth_gen"]["selected_tests_to_run"], {'jstests/auth/auth3.js'}) @patch(ns("_exclude_task")) @patch(ns("_find_task")) def test_get_config_for_test_mapping_when_task_should_be_excluded(self, find_task_mock, exclude_task_mock): find_task_mock.return_value = build_mock_evg_task( "jsCore_auth", "run tests", "--suites=core_auth --storageEngine=wiredTiger") exclude_task_mock.return_value = True tests_by_task = { "jsCore_auth": TaskInfo( display_task_name="task 1", tests=[ "jstests/core/currentop_waiting_for_latch.js", "jstests/core/latch_analyzer.js", ], resmoke_args="", require_multiversion=None, distro="", ), } task_config_service = under_test.TaskConfigService() task_configs = task_config_service.get_task_configs_for_test_mappings( tests_by_task, MagicMock()) self.assertEqual(task_configs, {}) @patch(ns("_find_task")) def test_get_config_for_test_mapping_when_task_does_not_exist(self, find_task_mock): find_task_mock.return_value = None tests_by_task = { "jsCore_auth": TaskInfo( display_task_name="task 1", tests=[ "jstests/core/currentop_waiting_for_latch.js", "jstests/core/latch_analyzer.js", ], resmoke_args="", require_multiversion=None, distro="", ), } task_config_service = under_test.TaskConfigService() task_configs = task_config_service.get_task_configs_for_test_mappings( tests_by_task, MagicMock()) self.assertEqual(task_configs, {}) class TestGetTaskConfigsForTaskMappings(unittest.TestCase): @patch(ns("_exclude_task")) @patch(ns("_find_task")) def test_get_config_for_task_mapping(self, find_task_mock, exclude_task_mock): find_task_mock.side_effect = [build_mock_evg_task("task_1"), build_mock_evg_task("task_2")] exclude_task_mock.return_value = False tasks = ["task_1", "task_2"] task_config_service = under_test.TaskConfigService() task_configs = task_config_service.get_task_configs_for_task_mappings(tasks, MagicMock()) self.assertEqual(task_configs["task_1"]["resmoke_args"], "--storageEngine=wiredTiger") self.assertEqual(task_configs["task_2"]["resmoke_args"], "--storageEngine=wiredTiger") @patch(ns("_exclude_task")) @patch(ns("_find_task")) def test_get_config_for_task_mapping_when_task_should_be_excluded(self, find_task_mock, exclude_task_mock): find_task_mock.return_value = build_mock_evg_task("task_1") exclude_task_mock.return_value = True tasks = ["task_1"] task_config_service = under_test.TaskConfigService() task_configs = task_config_service.get_task_configs_for_task_mappings(tasks, MagicMock()) self.assertEqual(task_configs, {}) @patch(ns("_find_task")) def test_get_config_for_task_mapping_when_task_does_not_exist(self, find_task_mock): find_task_mock.return_value = None tasks = ["task_1"] task_config_service = under_test.TaskConfigService() task_configs = task_config_service.get_task_configs_for_task_mappings(tasks, MagicMock()) self.assertEqual(task_configs, {}) class TestRemoveRepoPathPrefix(unittest.TestCase): def test_path_handling(self): filepath = under_test._remove_repo_path_prefix("sample_directory/src/file1.cpp") self.assertEqual(filepath, "sample_directory/src/file1.cpp")