mongo/buildscripts/tests/timeouts/test_timeout_service.py

"""Unit tests for timeout_service.py."""
import random
import unittest
from datetime import datetime, timedelta
from unittest.mock import MagicMock

from requests.exceptions import HTTPError
from evergreen import EvergreenApi

import buildscripts.timeouts.timeout_service as under_test
from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
from buildscripts.util.teststats import HistoricTaskData

# pylint: disable=missing-docstring,no-self-use,invalid-name,protected-access


def build_mock_service(evg_api=None, resmoke_proxy=None):
    end_date = datetime.now()
    start_date = end_date - timedelta(weeks=2)
    timeout_settings = under_test.TimeoutSettings(
        end_date=end_date,
        start_date=start_date,
    )
    return under_test.TimeoutService(
        evg_api=evg_api if evg_api else MagicMock(spec_set=EvergreenApi),
        resmoke_proxy=resmoke_proxy if resmoke_proxy else MagicMock(spec_set=ResmokeProxyService),
        timeout_settings=timeout_settings)


def tst_stat_mock(file, duration, pass_count):
    return MagicMock(test_file=file, avg_duration_pass=duration, num_pass=pass_count)


class TestGetTimeoutEstimate(unittest.TestCase):
    def test_no_stats_should_return_default_timeout(self):
        mock_evg_api = MagicMock(spec_set=EvergreenApi)
        mock_evg_api.test_stats_by_project.return_value = []
        timeout_service = build_mock_service(evg_api=mock_evg_api)
        timeout_params = under_test.TimeoutParams(
            evg_project="my project",
            build_variant="bv",
            task_name="my task",
            suite_name="my suite",
            is_asan=False,
        )

        timeout = timeout_service.get_timeout_estimate(timeout_params)

        self.assertFalse(timeout.is_specified())

    def test_a_test_with_missing_history_should_cause_a_default_timeout(self):
        mock_evg_api = MagicMock(spec_set=EvergreenApi)
        test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(30)]
        mock_evg_api.test_stats_by_project.return_value = test_stats
        mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService)
        mock_resmoke_proxy.list_tests.return_value = ["test_with_no_stats.js"]
        timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy)
        timeout_params = under_test.TimeoutParams(
            evg_project="my project",
            build_variant="bv",
            task_name="my task",
            suite_name="my suite",
            is_asan=False,
        )

        timeout = timeout_service.get_timeout_estimate(timeout_params)

        self.assertFalse(timeout.is_specified())

    def test_a_test_with_zero_runtime_history_should_cause_a_default_timeout(self):
        mock_evg_api = MagicMock(spec_set=EvergreenApi)
        test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(30)]
        test_stats.append(tst_stat_mock("zero.js", 0.0, 1))
        mock_evg_api.test_stats_by_project.return_value = test_stats
        mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService)
        mock_resmoke_proxy.list_tests.return_value = [ts.test_file for ts in test_stats]
        timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy)
        timeout_params = under_test.TimeoutParams(
            evg_project="my project",
            build_variant="bv",
            task_name="my task",
            suite_name="my suite",
            is_asan=False,
        )

        timeout = timeout_service.get_timeout_estimate(timeout_params)

        self.assertFalse(timeout.is_specified())

    def test_all_tests_with_runtime_history_should_use_custom_timeout(self):
        mock_evg_api = MagicMock(spec_set=EvergreenApi)
        n_tests = 30
        test_runtime = 600
        test_stats = [tst_stat_mock(f"test_{i}.js", test_runtime, 1) for i in range(n_tests)]
        mock_evg_api.test_stats_by_project.return_value = test_stats
        mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService)
        mock_resmoke_proxy.list_tests.return_value = [ts.test_file for ts in test_stats]
        timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy)
        timeout_params = under_test.TimeoutParams(
            evg_project="my project",
            build_variant="bv",
            task_name="my task",
            suite_name="my suite",
            is_asan=False,
        )

        timeout = timeout_service.get_timeout_estimate(timeout_params)

        self.assertTrue(timeout.is_specified())
        self.assertEqual(1860, timeout.calculate_test_timeout(1))
        self.assertEqual(54180, timeout.calculate_task_timeout(1))


class TestGetTaskHookOverhead(unittest.TestCase):
    def test_no_stats_should_return_zero(self):
        timeout_service = build_mock_service()

        overhead = timeout_service.get_task_hook_overhead("suite", is_asan=False, test_count=30,
                                                          historic_stats=None)

        self.assertEqual(0.0, overhead)

    def test_stats_with_no_clean_every_n_should_return_zero(self):
        timeout_service = build_mock_service()
        test_stats = HistoricTaskData.from_stats_list(
            [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(30)])

        overhead = timeout_service.get_task_hook_overhead("suite", is_asan=False, test_count=30,
                                                          historic_stats=test_stats)

        self.assertEqual(0.0, overhead)

    def test_stats_with_clean_every_n_should_return_overhead(self):
        test_count = 30
        runtime = 25
        timeout_service = build_mock_service()
        test_stat_list = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(test_count)]
        test_stat_list.extend([
            tst_stat_mock(f"test_{i}:{under_test.CLEAN_EVERY_N_HOOK}", runtime, 1)
            for i in range(10)
        ])
        random.shuffle(test_stat_list)
        test_stats = HistoricTaskData.from_stats_list(test_stat_list)

        overhead = timeout_service.get_task_hook_overhead(
            "suite", is_asan=True, test_count=test_count, historic_stats=test_stats)

        self.assertEqual(runtime * test_count, overhead)


class TestLookupHistoricStats(unittest.TestCase):
    def test_no_stats_from_evergreen_should_return_none(self):
        mock_evg_api = MagicMock(spec_set=EvergreenApi)
        mock_evg_api.test_stats_by_project.return_value = []
        timeout_service = build_mock_service(evg_api=mock_evg_api)
        timeout_params = under_test.TimeoutParams(
            evg_project="my project",
            build_variant="bv",
            task_name="my task",
            suite_name="my suite",
            is_asan=False,
        )

        stats = timeout_service.lookup_historic_stats(timeout_params)

        self.assertIsNone(stats)

    def test_errors_from_evergreen_should_return_none(self):
        mock_evg_api = MagicMock(spec_set=EvergreenApi)
        mock_evg_api.test_stats_by_project.side_effect = HTTPError("failed to connect")
        timeout_service = build_mock_service(evg_api=mock_evg_api)
        timeout_params = under_test.TimeoutParams(
            evg_project="my project",
            build_variant="bv",
            task_name="my task",
            suite_name="my suite",
            is_asan=False,
        )

        stats = timeout_service.lookup_historic_stats(timeout_params)

        self.assertIsNone(stats)

    def test_stats_from_evergreen_should_return_the_stats(self):
        mock_evg_api = MagicMock(spec_set=EvergreenApi)
        test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(100)]
        mock_evg_api.test_stats_by_project.return_value = test_stats
        timeout_service = build_mock_service(evg_api=mock_evg_api)
        timeout_params = under_test.TimeoutParams(
            evg_project="my project",
            build_variant="bv",
            task_name="my task",
            suite_name="my suite",
            is_asan=False,
        )

        stats = timeout_service.lookup_historic_stats(timeout_params)

        self.assertIsNotNone(stats)
        self.assertEqual(len(test_stats), len(stats.historic_test_results))


class TestGetCleanEveryNCadence(unittest.TestCase):
    def test_clean_every_n_cadence_on_asan(self):
        timeout_service = build_mock_service()

        cadence = timeout_service._get_clean_every_n_cadence("suite", True)

        self.assertEqual(1, cadence)

    def test_clean_every_n_cadence_from_hook_config(self):
        expected_n = 42
        mock_resmoke_proxy = MagicMock()
        mock_resmoke_proxy.read_suite_config.return_value = {
            "executor": {
                "hooks": [{
                    "class": "hook1",
                }, {
                    "class": under_test.CLEAN_EVERY_N_HOOK,
                    "n": expected_n,
                }]
            }
        }
        timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)

        cadence = timeout_service._get_clean_every_n_cadence("suite", False)

        self.assertEqual(expected_n, cadence)

    def test_clean_every_n_cadence_no_n_in_hook_config(self):
        mock_resmoke_proxy = MagicMock()
        mock_resmoke_proxy.read_suite_config.return_value = {
            "executor": {
                "hooks": [{
                    "class": "hook1",
                }, {
                    "class": under_test.CLEAN_EVERY_N_HOOK,
                }]
            }
        }
        timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)

        cadence = timeout_service._get_clean_every_n_cadence("suite", False)

        self.assertEqual(1, cadence)

    def test_clean_every_n_cadence_no_hook_config(self):
        mock_resmoke_proxy = MagicMock()
        mock_resmoke_proxy.read_suite_config.return_value = {
            "executor": {"hooks": [{
                "class": "hook1",
            }, ]}
        }
        timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)

        cadence = timeout_service._get_clean_every_n_cadence("suite", False)

        self.assertEqual(1, cadence)