Replace unittest.mock + aynctest with backported mock to suppor…

…t Python 3.7–3.11

The root of the conflict between 3.7 and 3.11 is `asynctest`, which is intensively used in the unit tests.

Python 3.11 removed `@asyncio.coroutine` (it was issuing warnings since Python 3.8). As a result, `asynctest` is completely broken in 3.11. The latest commit of `asynctest` is 3 years old (Nov 2019; now is Nov 2022), so `asynctest` can be deemed unmaintained.

The closest equivalent of `asynctest` is `unittest.mock.AsyncMock`, but it is available in Python 3.8 only. Python 3.7 can be supported with a back-ported standalone library `mock`. But if we use it, we should use it consistently in all places, not mixed with the StdLib's `unittest.mock`.

An alternative would be to drop Python 3.7 support prematurely — 8 months before its official end of life (Jun 2023) — in order to support Python 3.11 now.

Signed-off-by: Sergey Vasilyev <nolar@nolar.info>

pytest.ini

@@ -1,4 +1,6 @@
 [pytest]
+; The standalone `mock` instead of stdlib `unittest.mock` is only for AsyncMock in Python 3.7.
+mock_use_standalone_module = true
 asyncio_mode = auto
 addopts =
     --strict-markers

requirements.txt

@@ -3,7 +3,6 @@
 -e .
 aresponses
 astpath[xpath]
-asynctest
 certbuilder
 certvalidator
 codecov
@@ -13,6 +12,9 @@ freezegun
 import-linter
 isort
 lxml
+# Generally, `unittest.mock` is enough, but it lacks `AsyncMock` for Py 3.7.
+# TODO: Once 3.7 is removed (Jun 2023), roll back to unittest.mock.
+mock
 # Mypy requires typed-ast, which is broken on PyPy 3.7 (could work in PyPy 3.8).
 mypy==0.982; implementation_name == "cpython"
 pre-commit

tests/admission/test_admission_server.py

@@ -1,7 +1,7 @@
 import contextlib
-from unittest.mock import Mock
 
 import pytest
+from mock import Mock
 
 import kopf
 from kopf._cogs.aiokits.aiovalues import Container

tests/admission/test_serving_handler_selection.py

@@ -1,6 +1,5 @@
-from unittest.mock import Mock
-
 import pytest
+from mock import Mock
 
 import kopf
 from kopf._cogs.structs.ids import HandlerId

tests/admission/test_serving_kwargs_passthrough.py

@@ -1,6 +1,5 @@
-from unittest.mock import Mock
-
 import pytest
+from mock import Mock
 
 import kopf
 from kopf._core.engines.admission import serve_admission_request

tests/apis/test_iterjsonlines.py

@@ -1,4 +1,4 @@
-import asynctest
+from mock import Mock
 
 from kopf._cogs.clients.api import iter_jsonlines
 
@@ -8,7 +8,7 @@ async def iter_chunked(n: int):
         if False:  # to make this function a generator
             yield b''
 
-    content = asynctest.Mock(iter_chunked=iter_chunked)
+    content = Mock(iter_chunked=iter_chunked)
     lines = []
     async for line in iter_jsonlines(content):
         lines.append(line)
@@ -20,7 +20,7 @@ async def test_empty_chunk():
     async def iter_chunked(n: int):
         yield b''
 
-    content = asynctest.Mock(iter_chunked=iter_chunked)
+    content = Mock(iter_chunked=iter_chunked)
     lines = []
     async for line in iter_jsonlines(content):
         lines.append(line)
@@ -32,7 +32,7 @@ async def test_one_chunk_one_line():
     async def iter_chunked(n: int):
         yield b'hello'
 
-    content = asynctest.Mock(iter_chunked=iter_chunked)
+    content = Mock(iter_chunked=iter_chunked)
     lines = []
     async for line in iter_jsonlines(content):
         lines.append(line)
@@ -44,7 +44,7 @@ async def test_one_chunk_two_lines():
     async def iter_chunked(n: int):
         yield b'hello\nworld'
 
-    content = asynctest.Mock(iter_chunked=iter_chunked)
+    content = Mock(iter_chunked=iter_chunked)
     lines = []
     async for line in iter_jsonlines(content):
         lines.append(line)
@@ -56,7 +56,7 @@ async def test_one_chunk_empty_lines():
     async def iter_chunked(n: int):
         yield b'\n\nhello\n\nworld\n\n'
 
-    content = asynctest.Mock(iter_chunked=iter_chunked)
+    content = Mock(iter_chunked=iter_chunked)
     lines = []
     async for line in iter_jsonlines(content):
         lines.append(line)
@@ -70,7 +70,7 @@ async def iter_chunked(n: int):
         yield b'o\n\nwor'
         yield b'ld\n\n'
 
-    content = asynctest.Mock(iter_chunked=iter_chunked)
+    content = Mock(iter_chunked=iter_chunked)
     lines = []
     async for line in iter_jsonlines(content):
         lines.append(line)

tests/basic-structs/test_memories.py

@@ -1,4 +1,4 @@
-from unittest.mock import Mock
+from mock import Mock
 
 from kopf._cogs.structs.bodies import Body
 from kopf._cogs.structs.ephemera import Memo

tests/causation/test_kwargs.py

@@ -1,9 +1,9 @@
 import dataclasses
 import logging
 from typing import Type
-from unittest.mock import Mock
 
 import pytest
+from mock import Mock
 
 from kopf._cogs.configs.configuration import OperatorSettings
 from kopf._cogs.structs import diffs

tests/conftest.py

@@ -8,12 +8,10 @@
 import sys
 import time
 from typing import Set
-from unittest.mock import Mock
 
 import aiohttp.web
-import asynctest
 import pytest
-import pytest_mock
+from mock import AsyncMock, Mock
 
 import kopf
 from kopf._cogs.clients.auth import APIContext
@@ -33,7 +31,6 @@ def pytest_configure(config):
     config.addinivalue_line('filterwarnings', 'error')
 
     # Warnings from the testing tools out of our control should not fail the tests.
-    config.addinivalue_line('filterwarnings', 'ignore:"@coroutine":DeprecationWarning:asynctest.mock')
     config.addinivalue_line('filterwarnings', 'ignore:The loop argument:DeprecationWarning:aiohttp')
     config.addinivalue_line('filterwarnings', 'ignore:The loop argument:DeprecationWarning:asyncio')
     config.addinivalue_line('filterwarnings', 'ignore:is deprecated, use current_thread:DeprecationWarning:threading')
@@ -75,15 +72,6 @@ def _is_e2e(item):
         items[:] = etc + e2e
 
 
-# Substitute the regular mock with the async-aware mock in the `mocker` fixture.
-@pytest.fixture(scope='session', autouse=True)
-def enforce_asyncio_mocker(pytestconfig):
-    pytest_mock.plugin.get_mock_module = lambda config: asynctest
-    pytest_mock.get_mock_module = pytest_mock.plugin.get_mock_module
-    fixture = pytest_mock.MockerFixture(pytestconfig)
-    assert fixture.mock_module is asynctest, "Mock replacement failed!"
-
-
 @pytest.fixture(params=[
     ('kopf.dev', 'v1', 'kopfpeerings', True),
     ('zalando.org', 'v1', 'kopfpeerings', True),
@@ -283,7 +271,7 @@ def test_me(resp_mocker):
             assert callback.call_count == 1
     """
     def resp_maker(*args, **kwargs):
-        actual_response = asynctest.MagicMock(*args, **kwargs)
+        actual_response = AsyncMock(*args, **kwargs)
         async def resp_mock_effect(request):
             nonlocal actual_response
 
@@ -300,7 +288,7 @@ async def resp_mock_effect(request):
                 response = await response
             return response
 
-        return asynctest.CoroutineMock(side_effect=resp_mock_effect)
+        return AsyncMock(side_effect=resp_mock_effect)
     return resp_maker
 
 

tests/handling/conftest.py

@@ -35,9 +35,9 @@
 """
 import dataclasses
 from typing import Callable
-from unittest.mock import Mock
 
 import pytest
+from mock import Mock
 
 import kopf
 from kopf._core.intents.causes import ChangingCause

tests/handling/daemons/conftest.py

@@ -1,10 +1,10 @@
 import asyncio
 import contextlib
 import time
-import unittest.mock
 
 import freezegun
 import pytest
+from mock import MagicMock, patch
 
 import kopf
 from kopf._cogs.aiokits.aiotoggles import ToggleSet
@@ -19,7 +19,7 @@ class DaemonDummy:
 
     def __init__(self):
         super().__init__()
-        self.mock = unittest.mock.MagicMock()
+        self.mock = MagicMock()
         self.kwargs = {}
         self.steps = {
             'called': asyncio.Event(),
@@ -107,8 +107,7 @@ def frozen_time():
     with freezegun.freeze_time("2020-01-01 00:00:00") as frozen:
         # Use freezegun-supported time instead of system clocks -- for testing purposes only.
         # NB: Patch strictly after the time is frozen -- to use fake_time(), not real time().
-        with unittest.mock.patch('time.monotonic', time.time), \
-             unittest.mock.patch('time.perf_counter', time.time):
+        with patch('time.monotonic', time.time), patch('time.perf_counter', time.time):
             yield frozen
 
 

tests/handling/subhandling/test_subhandling.py

@@ -1,8 +1,8 @@
 import asyncio
 import logging
-from unittest.mock import Mock
 
 import pytest
+from mock import Mock
 
 import kopf
 from kopf._cogs.structs.ephemera import Memo

tests/handling/test_parametrization.py

@@ -1,5 +1,6 @@
 import asyncio
-from unittest.mock import Mock
+
+from mock import Mock
 
 import kopf
 from kopf._cogs.structs.ephemera import Memo

tests/hierarchies/conftest.py

@@ -1,6 +1,5 @@
-from unittest.mock import Mock
-
 import pytest
+from mock import Mock
 
 
 class CustomIterable:

tests/hierarchies/test_owner_referencing.py

@@ -1,7 +1,7 @@
 import copy
-from unittest.mock import call
 
 import pytest
+from mock import call
 
 import kopf
 from kopf._cogs.structs.bodies import Body, RawBody, RawMeta

tests/invocations/test_callbacks.py

@@ -3,7 +3,7 @@
 import traceback
 
 import pytest
-from asynctest import MagicMock
+from mock import Mock
 
 from kopf._cogs.structs.bodies import Body
 from kopf._cogs.structs.patches import Patch
@@ -32,6 +32,14 @@ async def async_fn(*args, **kwargs):
     return _find_marker()
 
 
+def sync_mock_fn(mock, *args, **kwargs):
+    return mock(*args, **kwargs)
+
+
+async def async_mock_fn(mock, *args, **kwargs):
+    return mock(*args, **kwargs)
+
+
 def partials(fn, n):
     partial = fn
     for _ in range(n):
@@ -79,8 +87,8 @@ async def wrapper(*args, wrapper=wrapper, **kwargs):
 
 fns = pytest.mark.parametrize(
     'fn', [
-        (sync_fn),
-        (async_fn),
+        (sync_mock_fn),
+        (async_mock_fn),
     ])
 
 # Every combination of partials, sync & async wrappers possible.
@@ -140,23 +148,23 @@ async def test_stacktrace_visibility(fn, expected):
 
 @fns
 async def test_result_returned(fn):
-    fn = MagicMock(fn, return_value=999)
-    result = await invoke(fn)
+    mock = Mock(return_value=999)
+    result = await invoke(fn, kwargs=dict(mock=mock))
     assert result == 999
 
 
 @fns
 async def test_explicit_args_passed_properly(fn):
-    fn = MagicMock(fn)
-    await invoke(fn, kwargs=dict(kw1=300, kw2=400))
+    mock = Mock()
+    await invoke(fn, kwargs=dict(mock=mock, kw1=300, kw2=400))
 
-    assert fn.called
-    assert fn.call_count == 1
+    assert mock.called
+    assert mock.call_count == 1
 
-    assert len(fn.call_args[0]) == 0
-    assert len(fn.call_args[1]) >= 2  # also the magic kwargs
-    assert fn.call_args[1]['kw1'] == 300
-    assert fn.call_args[1]['kw2'] == 400
+    assert len(mock.call_args[0]) == 0
+    assert len(mock.call_args[1]) >= 2  # also the magic kwargs
+    assert mock.call_args[1]['kw1'] == 300
+    assert mock.call_args[1]['kw2'] == 400
 
 
 @fns
@@ -180,12 +188,12 @@ async def test_special_kwargs_added(fn, resource):
         new=object(),
     )
 
-    fn = MagicMock(fn)
-    await invoke(fn, kwargsrc=cause)
+    mock = Mock()
+    await invoke(fn, kwargs=dict(mock=mock), kwargsrc=cause)
 
-    assert fn.called
-    assert fn.call_count == 1
+    assert mock.called
+    assert mock.call_count == 1
 
     # Only check that kwargs are passed at all. The exact kwargs per cause are tested separately.
-    assert 'logger' in fn.call_args[1]
-    assert 'resource' in fn.call_args[1]
+    assert 'logger' in mock.call_args[1]
+    assert 'resource' in mock.call_args[1]

tests/persistence/test_states.py

@@ -1,8 +1,8 @@
 import datetime
-from unittest.mock import Mock
 
 import freezegun
 import pytest
+from mock import Mock
 
 from kopf._cogs.configs.progress import SmartProgressStorage, StatusProgressStorage
 from kopf._cogs.structs.bodies import Body

tests/reactor/conftest.py

@@ -2,7 +2,7 @@
 import functools
 
 import pytest
-from asynctest import CoroutineMock
+from mock import AsyncMock
 
 from kopf._cogs.clients.watching import infinite_watch
 from kopf._core.reactor.queueing import watcher, worker as original_worker
@@ -16,13 +16,13 @@ def _autouse_resp_mocker(resp_mocker):
 @pytest.fixture()
 def processor():
     """ A mock for processor -- to be checked if the handler has been called. """
-    return CoroutineMock()
+    return AsyncMock()
 
 
 @pytest.fixture()
 def worker_spy(mocker):
     """ Spy on the watcher: actually call it, but provide the mock-fields. """
-    spy = CoroutineMock(spec=original_worker, wraps=original_worker)
+    spy = AsyncMock(spec=original_worker, wraps=original_worker)
     return mocker.patch('kopf._core.reactor.queueing.worker', spy)