From 6b0354a543d75050d703f30532fc4bc32a17a11e Mon Sep 17 00:00:00 2001 From: Sergey Vasilyev Date: Tue, 1 Nov 2022 18:21:57 +0100 Subject: [PATCH 1/2] Preserve back-ported `mock` instead of `unittest.mock` Signed-off-by: Sergey Vasilyev --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 36ebb047..aa5059aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -85,4 +85,4 @@ repos: rev: v2.34.0 hooks: - id: pyupgrade - args: [--py37-plus] + args: [--py37-plus, --keep-mock] From 52f1f82b5a92406b51b75192e145d185a1de73c0 Mon Sep 17 00:00:00 2001 From: Sergey Vasilyev Date: Tue, 1 Nov 2022 17:48:26 +0100 Subject: [PATCH 2/2] Utilise uvloop automatically if present for CLI & synchronous `run()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `uvloop` is a drop-in replacement for asyncio's event loops, which promises 2x–2.5x faster execution of the same i/o code (http://magic.io/blog/uvloop-blazing-fast-python-networking/). There is no reason to ignore it if it is installed. For convenience, Kopf can be installed as `pip install kopf[uvloop]` to bring the mode as a dependency. The implicit activation only affects the CLI mode — i.e. `kopf run …`. To disable, uninstall `uvloop`. For in-code invocations —such as `kopf.run()` or `KopfRunner`— it will use the preinstalled event loop policy (or the default one). `uvloop` is not activated implicitly if Kopf is embedded into a custom application. Signed-off-by: Sergey Vasilyev --- .github/workflows/ci.yaml | 3 ++ .github/workflows/thorough.yaml | 3 ++ docs/embedding.rst | 48 +++++++++++++++++++ docs/install.rst | 4 ++ kopf/_kits/runner.py | 3 +- kopf/cli.py | 14 +++++- setup.py | 3 ++ tests/testing/test_runner.py | 18 +++++++ .../aiotasks/test_coro_cancellation.py | 2 +- 9 files changed, 94 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b19aaa26..5c7f0224 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -39,6 +39,9 @@ jobs: matrix: install-extras: [ "", "full-auth" ] python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + include: + - install-extras: "uvloop" + python-version: "3.11" name: Python ${{ matrix.python-version }} ${{ matrix.install-extras }} runs-on: ubuntu-22.04 timeout-minutes: 5 # usually 2-3 mins diff --git a/.github/workflows/thorough.yaml b/.github/workflows/thorough.yaml index 63c2933b..24223c66 100644 --- a/.github/workflows/thorough.yaml +++ b/.github/workflows/thorough.yaml @@ -43,6 +43,9 @@ jobs: matrix: install-extras: [ "", "full-auth" ] python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + include: + - install-extras: "uvloop" + python-version: "3.11" name: Python ${{ matrix.python-version }} ${{ matrix.install-extras }} runs-on: ubuntu-22.04 timeout-minutes: 5 # usually 2-3 mins diff --git a/docs/embedding.rst b/docs/embedding.rst index 1fca56c3..069f5c0d 100644 --- a/docs/embedding.rst +++ b/docs/embedding.rst @@ -76,6 +76,54 @@ See more on the asyncio event loops and _contexts_ in `Asyncio Policies`__. __ https://docs.python.org/3/library/asyncio-policy.html +.. _custom-event-loops: + + +Custom event loops +================== + +Kopf can run in any AsyncIO-compatible event loop. For example, uvloop `claims to be 2x–2.5x times faster`__ than asyncio. To run Kopf in uvloop, call it this way: + +__ http://magic.io/blog/uvloop-blazing-fast-python-networking/ + +.. code-block:: python + + import kopf + import uvloop + + def main(): + loop = uvloop.EventLoopPolicy().get_event_loop() + loop.run(kopf.operator()) + +Or this way: + +.. code-block:: python + + import kopf + import uvloop + + def main(): + kopf.run(loop=uvloop.EventLoopPolicy().new_event_loop()) + +Or this way: + +.. code-block:: python + + import kopf + import uvloop + + def main(): + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + kopf.run() + +Or any other way the event loop prescribes in its documentation. + +Kopf's CLI (i.e. :command:`kopf run`) will use uvloop by default if it is installed. To disable this implicit behaviour, either uninstall uvloop from Kopf's environment, or run Kopf explicitly from the code using the standard event loop. + +For convenience, Kopf can be installed as ``pip install kopf[uvloop]`` to enable this mode automatically. + +Kopf will never implicitly activate the custom event loops if it is called from the code, not from the CLI. + Multiple operators ================== diff --git a/docs/install.rst b/docs/install.rst index ee8d4600..bf0cb508 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -18,6 +18,10 @@ authentication beyond username+password, fixed tokens, or client SSL certs pip install kopf[full-auth] +If you want extra i/o performance under the hood, install it as (also see :ref:`custom-event-loops`):: + + pip install kopf[uvloop] + Unless you use the standalone mode, create a few Kopf-specific custom resources in the cluster:: diff --git a/kopf/_kits/runner.py b/kopf/_kits/runner.py index f81f7a08..3a4603aa 100644 --- a/kopf/_kits/runner.py +++ b/kopf/_kits/runner.py @@ -130,7 +130,8 @@ def _target(self) -> None: ctxobj = cli.CLIControls( registry=self.registry, settings=self.settings, - stop_flag=self._stop) + stop_flag=self._stop, + loop=loop) runner = click.testing.CliRunner() result = runner.invoke(cli.main, *self.args, **self.kwargs, obj=ctxobj) except BaseException as e: diff --git a/kopf/cli.py b/kopf/cli.py index fcb8b321..173d3509 100644 --- a/kopf/cli.py +++ b/kopf/cli.py @@ -1,3 +1,4 @@ +import asyncio import dataclasses import functools import os @@ -23,6 +24,7 @@ class CLIControls: vault: Optional[credentials.Vault] = None registry: Optional[registries.OperatorRegistry] = None settings: Optional[configuration.OperatorSettings] = None + loop: Optional[asyncio.AbstractEventLoop] = None class LogFormatParamType(click.Choice): @@ -63,8 +65,15 @@ def wrapper(verbose: bool, quiet: bool, debug: bool, auto_envvar_prefix='KOPF', )) @click.version_option(prog_name='kopf') -def main() -> None: - pass +@click.make_pass_decorator(CLIControls, ensure=True) +def main(__controls: CLIControls) -> None: + if __controls.loop is None: # the pure CLI use, not a KopfRunner or other code + try: + import uvloop + except ImportError: + pass + else: + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) @main.command() @@ -113,6 +122,7 @@ def run( stop_flag=__controls.stop_flag, ready_flag=__controls.ready_flag, vault=__controls.vault, + loop=__controls.loop, ) diff --git a/setup.py b/setup.py index 8117f9cf..4db2ee12 100644 --- a/setup.py +++ b/setup.py @@ -70,6 +70,9 @@ 'pykube-ng', # 4.90 MB 'kubernetes', # 40.0 MB (!) ], + 'uvloop': [ + 'uvloop', # 9.00 MB + ], 'dev': [ 'pyngrok', # 1.00 MB + downloaded binary 'oscrypto', # 2.80 MB (smaller than cryptography: 8.7 MB) diff --git a/tests/testing/test_runner.py b/tests/testing/test_runner.py index 73b77db4..a3524018 100644 --- a/tests/testing/test_runner.py +++ b/tests/testing/test_runner.py @@ -1,3 +1,5 @@ +import asyncio + import pytest from kopf._cogs.configs.configuration import OperatorSettings @@ -10,6 +12,22 @@ def no_config_needed(login_mocks): pass +@pytest.fixture(autouse=True, params=['default', 'uvloop']) +def _event_loop_policy(request): + original_policy = asyncio.get_event_loop_policy() + if request.param == 'default': + policy = asyncio.DefaultEventLoopPolicy() + elif request.param == 'uvloop': + uvloop = pytest.importorskip('uvloop') + policy = uvloop.EventLoopPolicy() + else: + raise RuntimeError(f"Unknown event loop type {request.param!r}") + + asyncio.set_event_loop_policy(policy) + yield + asyncio.set_event_loop_policy(original_policy) + + def test_command_invocation_works(): with KopfRunner(['--help']) as runner: pass diff --git a/tests/utilities/aiotasks/test_coro_cancellation.py b/tests/utilities/aiotasks/test_coro_cancellation.py index 6d40d523..f71ddc9b 100644 --- a/tests/utilities/aiotasks/test_coro_cancellation.py +++ b/tests/utilities/aiotasks/test_coro_cancellation.py @@ -12,7 +12,7 @@ async def f(mock): return mock() -def factory(loop, coro_or_mock): +def factory(loop, coro_or_mock, context=None): coro = coro_or_mock._mock_wraps if isinstance(coro_or_mock, AsyncMock) else coro_or_mock return asyncio.Task(coro, loop=loop)