From e425cf6d89d93c3e6eed10e863a9ab0e3b8ed1e9 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Tue, 31 Oct 2023 12:00:49 -0500
Subject: [PATCH 01/30] Fix test_https_proxy_unsupported_tls_in_tls on MacOS

---
 tests/test_proxy_functional.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 3de78e80665..40335bee365 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -4,6 +4,7 @@
 import os
 import pathlib
 import platform
+import ssl
 from re import match as match_regex
 from typing import Any
 from unittest import mock
@@ -161,10 +162,11 @@ async def test_secure_https_proxy_absolute_path(
 @secure_proxy_xfail(raises=AssertionError)
 @pytest.mark.parametrize("web_server_endpoint_type", ("https",))
 @pytest.mark.usefixtures("loop")
+@pytest.mark.skipif(ASYNCIO_SUPPORTS_TLS_IN_TLS, reason="asyncio supports TLS in TLS")
 async def test_https_proxy_unsupported_tls_in_tls(
-    client_ssl_ctx,
-    secure_proxy_url,
-    web_server_endpoint_type,
+    client_ssl_ctx: ssl.SSLContext,
+    secure_proxy_url: str,
+    web_server_endpoint_type: str,
 ) -> None:
     """Ensure connecting to TLS endpoints w/ HTTPS proxy needs patching.
 

From f95bdf08c7193e06d8e442e26d83bdb79afa8a62 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 11:12:39 -0600
Subject: [PATCH 02/30] adjust tests

---
 tests/test_proxy_functional.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 623dab3b9e7..559bf80b3cd 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -16,7 +16,7 @@
 
 import aiohttp
 from aiohttp import web
-from aiohttp.client_exceptions import ClientConnectionError, ClientProxyConnectionError
+from aiohttp.client_exceptions import ClientConnectionError
 from aiohttp.helpers import PY_310
 
 pytestmark = [
@@ -132,7 +132,10 @@ def _pretend_asyncio_supports_tls_in_tls(
     )
 
 
-@secure_proxy_xfail(raises=ClientProxyConnectionError)
+@pytest.mark.skipif(
+    not ASYNCIO_SUPPORTS_TLS_IN_TLS,
+    reason="asyncio on this python does not support TLS in TLS",
+)
 @pytest.mark.parametrize("web_server_endpoint_type", ("http", "https"))
 @pytest.mark.usefixtures("_pretend_asyncio_supports_tls_in_tls", "loop")
 async def test_secure_https_proxy_absolute_path(
@@ -159,10 +162,11 @@ async def test_secure_https_proxy_absolute_path(
     await conn.close()
 
 
-@secure_proxy_xfail(raises=AssertionError)
 @pytest.mark.parametrize("web_server_endpoint_type", ("https",))
 @pytest.mark.usefixtures("loop")
-@pytest.mark.skipif(ASYNCIO_SUPPORTS_TLS_IN_TLS, reason="asyncio supports TLS in TLS")
+@pytest.mark.skipif(
+    ASYNCIO_SUPPORTS_TLS_IN_TLS, reason="asyncio on this python supports TLS in TLS"
+)
 async def test_https_proxy_unsupported_tls_in_tls(
     client_ssl_ctx: ssl.SSLContext,
     secure_proxy_url: str,

From c62cd7958d4c9d29748e28e198b011e0dbeafac8 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 11:13:50 -0600
Subject: [PATCH 03/30] adjust tests

---
 tests/test_proxy_functional.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 559bf80b3cd..72f3787b83a 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -139,10 +139,10 @@ def _pretend_asyncio_supports_tls_in_tls(
 @pytest.mark.parametrize("web_server_endpoint_type", ("http", "https"))
 @pytest.mark.usefixtures("_pretend_asyncio_supports_tls_in_tls", "loop")
 async def test_secure_https_proxy_absolute_path(
-    client_ssl_ctx,
-    secure_proxy_url,
-    web_server_endpoint_url,
-    web_server_endpoint_payload,
+    client_ssl_ctx: ssl.SSLContext,
+    secure_proxy_url: str,
+    web_server_endpoint_url: str,
+    web_server_endpoint_payload: str,
 ) -> None:
     """Ensure HTTP(S) sites are accessible through a secure proxy."""
     conn = aiohttp.TCPConnector()

From d68b650feb180ceed73c50b4c0e52860d1638a6a Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 11:24:50 -0600
Subject: [PATCH 04/30] vierfy the proxy is actually running and we can connect

---
 tests/test_proxy_functional.py | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 72f3787b83a..054b4a6240b 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -4,6 +4,7 @@
 import os
 import pathlib
 import platform
+import socket
 import ssl
 from re import match as match_regex
 from typing import Any
@@ -47,6 +48,25 @@
 )
 
 
+async def verify_port_accepts_connections(port: int) -> bool:
+    """Verify that a given port accepts connections.
+
+    This is done by trying to connect to it.
+    """
+    loop = asyncio.get_event_loop()
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.setblocking(False)
+    try:
+        async with asyncio.timeout(0.5):
+            await loop.sock_connect(sock, ("127.0.0.1", port))
+    except (OSError, TimeoutError):
+        return False
+    finally:
+        if sock is not None:
+            sock.close()
+    return True
+
+
 @pytest.fixture
 def secure_proxy_url(tls_certificate_pem_path):
     """Return the URL of an instance of a running secure proxy.
@@ -140,7 +160,7 @@ def _pretend_asyncio_supports_tls_in_tls(
 @pytest.mark.usefixtures("_pretend_asyncio_supports_tls_in_tls", "loop")
 async def test_secure_https_proxy_absolute_path(
     client_ssl_ctx: ssl.SSLContext,
-    secure_proxy_url: str,
+    secure_proxy_url: URL,
     web_server_endpoint_url: str,
     web_server_endpoint_payload: str,
 ) -> None:
@@ -148,6 +168,11 @@ async def test_secure_https_proxy_absolute_path(
     conn = aiohttp.TCPConnector()
     sess = aiohttp.ClientSession(connector=conn)
 
+    # Verify the proxy is up and running.
+    assert await verify_port_accepts_connections(
+        secure_proxy_url.port
+    ), "Could not connect to proxy"
+
     response = await sess.get(
         web_server_endpoint_url,
         proxy=secure_proxy_url,
@@ -169,7 +194,7 @@ async def test_secure_https_proxy_absolute_path(
 )
 async def test_https_proxy_unsupported_tls_in_tls(
     client_ssl_ctx: ssl.SSLContext,
-    secure_proxy_url: str,
+    secure_proxy_url: URL,
     web_server_endpoint_type: str,
 ) -> None:
     """Ensure connecting to TLS endpoints w/ HTTPS proxy needs patching.
@@ -177,6 +202,11 @@ async def test_https_proxy_unsupported_tls_in_tls(
     This also checks that a helpful warning on how to patch the env
     is displayed.
     """
+    # Verify the proxy is up and running.
+    assert await verify_port_accepts_connections(
+        secure_proxy_url.port
+    ), "Could not connect to proxy"
+
     url = URL.build(scheme=web_server_endpoint_type, host="python.org")
 
     escaped_host_port = ":".join((url.host.replace(".", r"\."), str(url.port)))

From 0b1a604f27e976be4c7b11a254360182c33cc0af Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 11:29:56 -0600
Subject: [PATCH 05/30] legacy

---
 tests/test_proxy_functional.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 054b4a6240b..622c04101a5 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -11,6 +11,7 @@
 from unittest import mock
 from uuid import uuid4
 
+import async_timeout
 import proxy
 import pytest
 from yarl import URL
@@ -57,7 +58,7 @@ async def verify_port_accepts_connections(port: int) -> bool:
     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     sock.setblocking(False)
     try:
-        async with asyncio.timeout(0.5):
+        async with async_timeout.timeout(0.5):
             await loop.sock_connect(sock, ("127.0.0.1", port))
     except (OSError, TimeoutError):
         return False

From 8bde66f92d53e77881cdb0b27fc2a1f0297bae9b Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 11:39:00 -0600
Subject: [PATCH 06/30] cond

---
 tests/test_proxy_functional.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 622c04101a5..b2d4bdb27bc 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -6,12 +6,12 @@
 import platform
 import socket
 import ssl
+import sys
 from re import match as match_regex
 from typing import Any
 from unittest import mock
 from uuid import uuid4
 
-import async_timeout
 import proxy
 import pytest
 from yarl import URL
@@ -21,6 +21,11 @@
 from aiohttp.client_exceptions import ClientConnectionError
 from aiohttp.helpers import PY_310
 
+if sys.version_info >= (3, 11):
+    import asyncio as async_timeout
+else:
+    import async_timeout
+
 pytestmark = [
     pytest.mark.filterwarnings(
         "ignore:unclosed <socket.socket fd=.*:ResourceWarning",

From 2e5972deb484b0fd590c7c9fd532af15e7f30a7c Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 11:46:27 -0600
Subject: [PATCH 07/30] adjust workers

---
 tests/test_proxy_functional.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index b2d4bdb27bc..ddd94f31130 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -82,7 +82,7 @@ def secure_proxy_url(tls_certificate_pem_path):
     proxypy_args = [
         "--threadless",  # use asyncio
         "--num-workers",
-        "1",  # the tests only send one query anyway
+        "3",  # ensure we can handle multiple connections
         "--hostname",
         "127.0.0.1",  # network interface to listen to
         "--port",

From e467cb9fc963a97dde67497c4f13f4a48fd4367f Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 11:57:08 -0600
Subject: [PATCH 08/30] try disabling verify

---
 tests/test_proxy_functional.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index ddd94f31130..fb4ad5bd983 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -218,7 +218,11 @@ async def test_https_proxy_unsupported_tls_in_tls(
     escaped_host_port = ":".join((url.host.replace(".", r"\."), str(url.port)))
     escaped_proxy_url = str(secure_proxy_url).replace(".", r"\.")
 
-    conn = aiohttp.TCPConnector()
+    ssl_context = ssl.SSLContext()
+    ssl_context.check_hostname = False
+    ssl_context.verify_mode = ssl.CERT_NONE
+
+    conn = aiohttp.TCPConnector(ssl=ssl_context)
     sess = aiohttp.ClientSession(connector=conn)
 
     expected_warning_text = (

From 9f97902a0dc412d72d96040257c1610f62a10a42 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 11:59:05 -0600
Subject: [PATCH 09/30] debug logging

---
 tests/test_proxy_functional.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index fb4ad5bd983..2e60195fcf3 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -91,6 +91,8 @@ def secure_proxy_url(tls_certificate_pem_path):
         tls_certificate_pem_path,  # contains both key and cert
         "--key-file",
         tls_certificate_pem_path,  # contains both key and cert
+        "--log-level",
+        "d",
     ]
 
     with proxy.Proxy(input_args=proxypy_args) as proxy_instance:

From e708df232b6177a4bf6bf6e54e1ef22ed2989727 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 12:21:06 -0600
Subject: [PATCH 10/30] debug

---
 tests/test_proxy_functional.py | 22 +++-------------------
 1 file changed, 3 insertions(+), 19 deletions(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 2e60195fcf3..1d21b1465d0 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -1,6 +1,7 @@
 # type: ignore
 import asyncio
 import functools
+import logging
 import os
 import pathlib
 import platform
@@ -95,6 +96,8 @@ def secure_proxy_url(tls_certificate_pem_path):
         "d",
     ]
 
+    logging.getLogger("proxy.proxy").setLevel(logging.DEBUG)
+
     with proxy.Proxy(input_args=proxypy_args) as proxy_instance:
         yield URL.build(
             scheme="https",
@@ -142,30 +145,11 @@ async def handler(*args, **kwargs):
     )
 
 
-@pytest.fixture
-def _pretend_asyncio_supports_tls_in_tls(
-    monkeypatch,
-    web_server_endpoint_type,
-):
-    if web_server_endpoint_type != "https" or ASYNCIO_SUPPORTS_TLS_IN_TLS:
-        return
-
-    # for https://github.com/python/cpython/pull/28073
-    # and https://bugs.python.org/issue37179
-    monkeypatch.setattr(
-        asyncio.sslproto._SSLProtocolTransport,
-        "_start_tls_compatible",
-        True,
-        raising=False,
-    )
-
-
 @pytest.mark.skipif(
     not ASYNCIO_SUPPORTS_TLS_IN_TLS,
     reason="asyncio on this python does not support TLS in TLS",
 )
 @pytest.mark.parametrize("web_server_endpoint_type", ("http", "https"))
-@pytest.mark.usefixtures("_pretend_asyncio_supports_tls_in_tls", "loop")
 async def test_secure_https_proxy_absolute_path(
     client_ssl_ctx: ssl.SSLContext,
     secure_proxy_url: URL,

From b29790206b7e3c0ec697acc70fe9be62ccfe86f5 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 13:07:07 -0600
Subject: [PATCH 11/30] fix lgoger

---
 tests/test_proxy_functional.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 1d21b1465d0..d5df0f7cad6 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -96,8 +96,7 @@ def secure_proxy_url(tls_certificate_pem_path):
         "d",
     ]
 
-    logging.getLogger("proxy.proxy").setLevel(logging.DEBUG)
-
+    logging.getLogger("proxy").setLevel(logging.DEBUG)
     with proxy.Proxy(input_args=proxypy_args) as proxy_instance:
         yield URL.build(
             scheme="https",

From 62a279bfe81d43afc1eb745add36757171ddcaf8 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 13:15:50 -0600
Subject: [PATCH 12/30] fix lgoger

---
 .github/workflows/ci-cd.yml    | 2 +-
 tests/test_proxy_functional.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
index e961250bf67..7b89f67bbf0 100644
--- a/.github/workflows/ci-cd.yml
+++ b/.github/workflows/ci-cd.yml
@@ -241,7 +241,7 @@ jobs:
         COLOR: yes
         AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
       run: >-  # `exit 1` makes sure that the job remains red with flaky runs
-        pytest --no-cov -vvvvv --lf && exit 1
+        pytest --no-cov -vvvvv -s --lf && exit 1
       shell: bash
     - name: Run dev_mode tests
       env:
diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index d5df0f7cad6..032246c2051 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -176,6 +176,7 @@ async def test_secure_https_proxy_absolute_path(
     response.close()
     await sess.close()
     await conn.close()
+    raise NotImplementedError("TODO: test HTTPS proxy")
 
 
 @pytest.mark.parametrize("web_server_endpoint_type", ("https",))

From 6b9159001920cad0377de26ab979469ce547037d Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 13:37:54 -0600
Subject: [PATCH 13/30] print log

---
 tests/test_proxy_functional.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 032246c2051..f172930b3eb 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -8,6 +8,7 @@
 import socket
 import ssl
 import sys
+from pathlib import Path
 from re import match as match_regex
 from typing import Any
 from unittest import mock
@@ -75,11 +76,12 @@ async def verify_port_accepts_connections(port: int) -> bool:
 
 
 @pytest.fixture
-def secure_proxy_url(tls_certificate_pem_path):
+def secure_proxy_url(tls_certificate_pem_path, tmp_path: Path):
     """Return the URL of an instance of a running secure proxy.
 
     This fixture also spawns that instance and tears it down after the test.
     """
+    log_path = tmp_path.joinpath("proxy.log")
     proxypy_args = [
         "--threadless",  # use asyncio
         "--num-workers",
@@ -94,8 +96,11 @@ def secure_proxy_url(tls_certificate_pem_path):
         tls_certificate_pem_path,  # contains both key and cert
         "--log-level",
         "d",
+        "--log-file",
+        str(log_path),
     ]
 
+    logging.getLogger().setLevel(logging.DEBUG)
     logging.getLogger("proxy").setLevel(logging.DEBUG)
     with proxy.Proxy(input_args=proxypy_args) as proxy_instance:
         yield URL.build(
@@ -103,6 +108,10 @@ def secure_proxy_url(tls_certificate_pem_path):
             host=str(proxy_instance.flags.hostname),
             port=proxy_instance.flags.port,
         )
+        with open(log_path) as log_file:
+            print("proxy log:")
+            log_contents = log_file.read()
+            print(log_contents)
 
 
 @pytest.fixture

From f90eb898b2c3e4e4725440a511ae723fc3ddcf6b Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 13:47:31 -0600
Subject: [PATCH 14/30] more info

---
 tests/test_proxy_functional.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index f172930b3eb..045da33ade5 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -103,6 +103,7 @@ def secure_proxy_url(tls_certificate_pem_path, tmp_path: Path):
     logging.getLogger().setLevel(logging.DEBUG)
     logging.getLogger("proxy").setLevel(logging.DEBUG)
     with proxy.Proxy(input_args=proxypy_args) as proxy_instance:
+        print(proxy_instance.flags.__dict__)
         yield URL.build(
             scheme="https",
             host=str(proxy_instance.flags.hostname),

From 05e7afd280892318aee1d1f13d93bb26fa1f1c30 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 13:54:52 -0600
Subject: [PATCH 15/30] more info

---
 tests/test_proxy_functional.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 045da33ade5..f797a79b727 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -109,10 +109,13 @@ def secure_proxy_url(tls_certificate_pem_path, tmp_path: Path):
             host=str(proxy_instance.flags.hostname),
             port=proxy_instance.flags.port,
         )
-        with open(log_path) as log_file:
-            print("proxy log:")
-            log_contents = log_file.read()
-            print(log_contents)
+        try:
+            with open(log_path) as log_file:
+                print("proxy log:")
+                log_contents = log_file.read()
+                print(log_contents)
+        except FileNotFoundError:
+            print("proxy log not found")
 
 
 @pytest.fixture

From d0e68a479cfb92080b554951b02016b5557d2a9c Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 14:06:13 -0600
Subject: [PATCH 16/30] proxy seems to die on tls connect

---
 tests/test_proxy_functional.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index f797a79b727..c1097aef27b 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -100,6 +100,11 @@ def secure_proxy_url(tls_certificate_pem_path, tmp_path: Path):
         str(log_path),
     ]
 
+    with open(tls_certificate_pem_path) as tls_certificate_pem_file:
+        tls_certificate_pem = tls_certificate_pem_file.read()
+        print("tls_certificate_pem:")
+        print(tls_certificate_pem)
+
     logging.getLogger().setLevel(logging.DEBUG)
     logging.getLogger("proxy").setLevel(logging.DEBUG)
     with proxy.Proxy(input_args=proxypy_args) as proxy_instance:

From 8098b75f827db00d619dd5e505d02585ca65ddcc Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 14:50:55 -0600
Subject: [PATCH 17/30] try rsa

---
 tests/conftest.py              | 8 +++++---
 tests/test_proxy_functional.py | 6 ------
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 3c3389500fb..62dd3ac2acf 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -19,7 +19,7 @@
     import trustme
 
     # Check if the CA is available in runtime, MacOS on Py3.10 fails somehow
-    trustme.CA()
+    trustme.CA(key_type=trustme.KeyType.RSA)
 
     TRUSTME: bool = True
 except ImportError:
@@ -35,16 +35,17 @@
 def tls_certificate_authority() -> Any:
     if not TRUSTME:
         pytest.xfail("trustme is not supported")
-    return trustme.CA()
+    return trustme.CA(key_type=trustme.KeyType.RSA)
 
 
 @pytest.fixture
-def tls_certificate(tls_certificate_authority: Any) -> Any:
+def tls_certificate(tls_certificate_authority: "trustme.CA") -> Any:
     return tls_certificate_authority.issue_cert(
         "localhost",
         "xn--prklad-4va.localhost",
         "127.0.0.1",
         "::1",
+        key_type=trustme.KeyType.RSA,
     )
 
 
@@ -65,6 +66,7 @@ def client_ssl_ctx(tls_certificate_authority: Any) -> ssl.SSLContext:
 @pytest.fixture
 def tls_ca_certificate_pem_path(tls_certificate_authority: Any) -> None:
     with tls_certificate_authority.cert_pem.tempfile() as ca_cert_pem:
+        print(ca_cert_pem)
         yield ca_cert_pem
 
 
diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index c1097aef27b..c1309631e68 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -100,11 +100,6 @@ def secure_proxy_url(tls_certificate_pem_path, tmp_path: Path):
         str(log_path),
     ]
 
-    with open(tls_certificate_pem_path) as tls_certificate_pem_file:
-        tls_certificate_pem = tls_certificate_pem_file.read()
-        print("tls_certificate_pem:")
-        print(tls_certificate_pem)
-
     logging.getLogger().setLevel(logging.DEBUG)
     logging.getLogger("proxy").setLevel(logging.DEBUG)
     with proxy.Proxy(input_args=proxypy_args) as proxy_instance:
@@ -194,7 +189,6 @@ async def test_secure_https_proxy_absolute_path(
     response.close()
     await sess.close()
     await conn.close()
-    raise NotImplementedError("TODO: test HTTPS proxy")
 
 
 @pytest.mark.parametrize("web_server_endpoint_type", ("https",))

From f361328700bc97f8e1e8540c9dd6d2c3b7a8bb7d Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 15 Nov 2023 14:57:44 -0600
Subject: [PATCH 18/30] maybe the proxy is dying off?

---
 tests/test_proxy_functional.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index c1309631e68..60a7a016125 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -177,11 +177,18 @@ async def test_secure_https_proxy_absolute_path(
         secure_proxy_url.port
     ), "Could not connect to proxy"
 
-    response = await sess.get(
-        web_server_endpoint_url,
-        proxy=secure_proxy_url,
-        ssl=client_ssl_ctx,  # used for both proxy and endpoint connections
-    )
+    try:
+        response = await sess.get(
+            web_server_endpoint_url,
+            proxy=secure_proxy_url,
+            ssl=client_ssl_ctx,  # used for both proxy and endpoint connections
+        )
+    except Exception:
+        # Verify the proxy is up and running.
+        assert await verify_port_accepts_connections(
+            secure_proxy_url.port
+        ), "Could not connect to proxy"
+        raise
 
     assert response.status == 200
     assert await response.text() == web_server_endpoint_payload

From d3567f51a66dd574e7d28fc1ea9e00464af436cb Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 19 Nov 2023 13:55:47 -0600
Subject: [PATCH 19/30] tweak

---
 tests/test_proxy_functional.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 60a7a016125..1995d96b75c 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -162,6 +162,7 @@ async def handler(*args, **kwargs):
     reason="asyncio on this python does not support TLS in TLS",
 )
 @pytest.mark.parametrize("web_server_endpoint_type", ("http", "https"))
+@pytest.mark.filterwarnings(r".*ssl.OP_NO_SSL*")
 async def test_secure_https_proxy_absolute_path(
     client_ssl_ctx: ssl.SSLContext,
     secure_proxy_url: URL,

From 24494378f2131e137eaed391a164c76893477e0c Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 19 Nov 2023 13:56:23 -0600
Subject: [PATCH 20/30] tweak

---
 tests/test_proxy_functional.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 1995d96b75c..bccf9079ad2 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -162,7 +162,7 @@ async def handler(*args, **kwargs):
     reason="asyncio on this python does not support TLS in TLS",
 )
 @pytest.mark.parametrize("web_server_endpoint_type", ("http", "https"))
-@pytest.mark.filterwarnings(r".*ssl.OP_NO_SSL*")
+@pytest.mark.filterwarnings(r"ignore:.*ssl.OP_NO_SSL*")
 async def test_secure_https_proxy_absolute_path(
     client_ssl_ctx: ssl.SSLContext,
     secure_proxy_url: URL,

From eca01beb757bb41f7981455d4f75b622f5fb39ea Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 19 Nov 2023 13:56:56 -0600
Subject: [PATCH 21/30] revet

---
 .github/workflows/ci-cd.yml | 2 +-
 tests/conftest.py           | 8 +++-----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
index 7b89f67bbf0..e961250bf67 100644
--- a/.github/workflows/ci-cd.yml
+++ b/.github/workflows/ci-cd.yml
@@ -241,7 +241,7 @@ jobs:
         COLOR: yes
         AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
       run: >-  # `exit 1` makes sure that the job remains red with flaky runs
-        pytest --no-cov -vvvvv -s --lf && exit 1
+        pytest --no-cov -vvvvv --lf && exit 1
       shell: bash
     - name: Run dev_mode tests
       env:
diff --git a/tests/conftest.py b/tests/conftest.py
index 62dd3ac2acf..3c3389500fb 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -19,7 +19,7 @@
     import trustme
 
     # Check if the CA is available in runtime, MacOS on Py3.10 fails somehow
-    trustme.CA(key_type=trustme.KeyType.RSA)
+    trustme.CA()
 
     TRUSTME: bool = True
 except ImportError:
@@ -35,17 +35,16 @@
 def tls_certificate_authority() -> Any:
     if not TRUSTME:
         pytest.xfail("trustme is not supported")
-    return trustme.CA(key_type=trustme.KeyType.RSA)
+    return trustme.CA()
 
 
 @pytest.fixture
-def tls_certificate(tls_certificate_authority: "trustme.CA") -> Any:
+def tls_certificate(tls_certificate_authority: Any) -> Any:
     return tls_certificate_authority.issue_cert(
         "localhost",
         "xn--prklad-4va.localhost",
         "127.0.0.1",
         "::1",
-        key_type=trustme.KeyType.RSA,
     )
 
 
@@ -66,7 +65,6 @@ def client_ssl_ctx(tls_certificate_authority: Any) -> ssl.SSLContext:
 @pytest.fixture
 def tls_ca_certificate_pem_path(tls_certificate_authority: Any) -> None:
     with tls_certificate_authority.cert_pem.tempfile() as ca_cert_pem:
-        print(ca_cert_pem)
         yield ca_cert_pem
 
 

From 9160506fe6cd38f6ff7f5f344640bf728b4d947d Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 19 Nov 2023 13:58:13 -0600
Subject: [PATCH 22/30] remove debug

---
 tests/test_proxy_functional.py | 69 +++-------------------------------
 1 file changed, 5 insertions(+), 64 deletions(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index bccf9079ad2..adf6b891e9b 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -1,13 +1,10 @@
 # type: ignore
 import asyncio
 import functools
-import logging
 import os
 import pathlib
 import platform
-import socket
 import ssl
-import sys
 from pathlib import Path
 from re import match as match_regex
 from typing import Any
@@ -23,11 +20,6 @@
 from aiohttp.client_exceptions import ClientConnectionError
 from aiohttp.helpers import PY_310
 
-if sys.version_info >= (3, 11):
-    import asyncio as async_timeout
-else:
-    import async_timeout
-
 pytestmark = [
     pytest.mark.filterwarnings(
         "ignore:unclosed <socket.socket fd=.*:ResourceWarning",
@@ -56,32 +48,12 @@
 )
 
 
-async def verify_port_accepts_connections(port: int) -> bool:
-    """Verify that a given port accepts connections.
-
-    This is done by trying to connect to it.
-    """
-    loop = asyncio.get_event_loop()
-    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    sock.setblocking(False)
-    try:
-        async with async_timeout.timeout(0.5):
-            await loop.sock_connect(sock, ("127.0.0.1", port))
-    except (OSError, TimeoutError):
-        return False
-    finally:
-        if sock is not None:
-            sock.close()
-    return True
-
-
 @pytest.fixture
 def secure_proxy_url(tls_certificate_pem_path, tmp_path: Path):
     """Return the URL of an instance of a running secure proxy.
 
     This fixture also spawns that instance and tears it down after the test.
     """
-    log_path = tmp_path.joinpath("proxy.log")
     proxypy_args = [
         "--threadless",  # use asyncio
         "--num-workers",
@@ -94,28 +66,14 @@ def secure_proxy_url(tls_certificate_pem_path, tmp_path: Path):
         tls_certificate_pem_path,  # contains both key and cert
         "--key-file",
         tls_certificate_pem_path,  # contains both key and cert
-        "--log-level",
-        "d",
-        "--log-file",
-        str(log_path),
     ]
 
-    logging.getLogger().setLevel(logging.DEBUG)
-    logging.getLogger("proxy").setLevel(logging.DEBUG)
     with proxy.Proxy(input_args=proxypy_args) as proxy_instance:
-        print(proxy_instance.flags.__dict__)
         yield URL.build(
             scheme="https",
             host=str(proxy_instance.flags.hostname),
             port=proxy_instance.flags.port,
         )
-        try:
-            with open(log_path) as log_file:
-                print("proxy log:")
-                log_contents = log_file.read()
-                print(log_contents)
-        except FileNotFoundError:
-            print("proxy log not found")
 
 
 @pytest.fixture
@@ -173,23 +131,11 @@ async def test_secure_https_proxy_absolute_path(
     conn = aiohttp.TCPConnector()
     sess = aiohttp.ClientSession(connector=conn)
 
-    # Verify the proxy is up and running.
-    assert await verify_port_accepts_connections(
-        secure_proxy_url.port
-    ), "Could not connect to proxy"
-
-    try:
-        response = await sess.get(
-            web_server_endpoint_url,
-            proxy=secure_proxy_url,
-            ssl=client_ssl_ctx,  # used for both proxy and endpoint connections
-        )
-    except Exception:
-        # Verify the proxy is up and running.
-        assert await verify_port_accepts_connections(
-            secure_proxy_url.port
-        ), "Could not connect to proxy"
-        raise
+    response = await sess.get(
+        web_server_endpoint_url,
+        proxy=secure_proxy_url,
+        ssl=client_ssl_ctx,  # used for both proxy and endpoint connections
+    )
 
     assert response.status == 200
     assert await response.text() == web_server_endpoint_payload
@@ -214,11 +160,6 @@ async def test_https_proxy_unsupported_tls_in_tls(
     This also checks that a helpful warning on how to patch the env
     is displayed.
     """
-    # Verify the proxy is up and running.
-    assert await verify_port_accepts_connections(
-        secure_proxy_url.port
-    ), "Could not connect to proxy"
-
     url = URL.build(scheme=web_server_endpoint_type, host="python.org")
 
     escaped_host_port = ":".join((url.host.replace(".", r"\."), str(url.port)))

From 2658afcfa3a3dc1181318a73cd4ce8e59e199fe5 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 19 Nov 2023 13:58:59 -0600
Subject: [PATCH 23/30] remove debug

---
 tests/test_proxy_functional.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index adf6b891e9b..2f73cb66e28 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -5,7 +5,6 @@
 import pathlib
 import platform
 import ssl
-from pathlib import Path
 from re import match as match_regex
 from typing import Any
 from unittest import mock
@@ -49,7 +48,7 @@
 
 
 @pytest.fixture
-def secure_proxy_url(tls_certificate_pem_path, tmp_path: Path):
+def secure_proxy_url(tls_certificate_pem_path):
     """Return the URL of an instance of a running secure proxy.
 
     This fixture also spawns that instance and tears it down after the test.
@@ -57,7 +56,7 @@ def secure_proxy_url(tls_certificate_pem_path, tmp_path: Path):
     proxypy_args = [
         "--threadless",  # use asyncio
         "--num-workers",
-        "3",  # ensure we can handle multiple connections
+        "1",  # the tests only send one query anyway
         "--hostname",
         "127.0.0.1",  # network interface to listen to
         "--port",

From cf216cb4b22af337842bf604b5d5eb17b701dc8f Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 19 Nov 2023 13:59:32 -0600
Subject: [PATCH 24/30] remove debug

---
 tests/test_proxy_functional.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 2f73cb66e28..2c3e514ead5 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -164,11 +164,7 @@ async def test_https_proxy_unsupported_tls_in_tls(
     escaped_host_port = ":".join((url.host.replace(".", r"\."), str(url.port)))
     escaped_proxy_url = str(secure_proxy_url).replace(".", r"\.")
 
-    ssl_context = ssl.SSLContext()
-    ssl_context.check_hostname = False
-    ssl_context.verify_mode = ssl.CERT_NONE
-
-    conn = aiohttp.TCPConnector(ssl=ssl_context)
+    conn = aiohttp.TCPConnector()
     sess = aiohttp.ClientSession(connector=conn)
 
     expected_warning_text = (

From c9aa93e01d50a0356ddcd675862b597fd1c02251 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 19 Nov 2023 14:02:31 -0600
Subject: [PATCH 25/30] fix

---
 tests/test_proxy_functional.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 2c3e514ead5..b7b88011ba8 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -120,6 +120,9 @@ async def handler(*args, **kwargs):
 )
 @pytest.mark.parametrize("web_server_endpoint_type", ("http", "https"))
 @pytest.mark.filterwarnings(r"ignore:.*ssl.OP_NO_SSL*")
+# Filter out the warning from
+# https://github.com/abhinavsingh/proxy.py/blob/30574fd0414005dfa8792a6e797023e862bdcf43/proxy/common/utils.py#L226
+# otherwise this test will fail because the proxy will die with an error.
 async def test_secure_https_proxy_absolute_path(
     client_ssl_ctx: ssl.SSLContext,
     secure_proxy_url: URL,

From c6a93a4fccb1647eee1117cdaedbbc40d4b35d53 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 19 Nov 2023 14:08:37 -0600
Subject: [PATCH 26/30] fix

---
 tests/test_proxy_functional.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index b7b88011ba8..fbfcbdfcc32 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -152,6 +152,10 @@ async def test_secure_https_proxy_absolute_path(
 @pytest.mark.skipif(
     ASYNCIO_SUPPORTS_TLS_IN_TLS, reason="asyncio on this python supports TLS in TLS"
 )
+@pytest.mark.filterwarnings(r"ignore:.*ssl.OP_NO_SSL*")
+# Filter out the warning from
+# https://github.com/abhinavsingh/proxy.py/blob/30574fd0414005dfa8792a6e797023e862bdcf43/proxy/common/utils.py#L226
+# otherwise this test will fail because the proxy will die with an error.
 async def test_https_proxy_unsupported_tls_in_tls(
     client_ssl_ctx: ssl.SSLContext,
     secure_proxy_url: URL,

From d0d57905c4dec1d4db16911085319927fc2e3793 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 22 Nov 2023 18:55:49 -0600
Subject: [PATCH 27/30] try threaded

---
 tests/test_proxy_functional.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index fbfcbdfcc32..07ec9049a5a 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -54,7 +54,7 @@ def secure_proxy_url(tls_certificate_pem_path):
     This fixture also spawns that instance and tears it down after the test.
     """
     proxypy_args = [
-        "--threadless",  # use asyncio
+        "--threaded",  # needed for windows
         "--num-workers",
         "1",  # the tests only send one query anyway
         "--hostname",

From 07f97bac630729fd06c3b63e1170e1262d44157a Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 22 Nov 2023 19:15:35 -0600
Subject: [PATCH 28/30] empty


From f6881fd4f250cdde310fda22d175d7553f149765 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Thu, 23 Nov 2023 05:48:04 -0600
Subject: [PATCH 29/30] cleanups

---
 tests/test_proxy_functional.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 07ec9049a5a..402a7615e53 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -5,6 +5,7 @@
 import pathlib
 import platform
 import ssl
+import sys
 from re import match as match_regex
 from typing import Any
 from unittest import mock
@@ -41,10 +42,7 @@
     ),
 )
 
-ASYNCIO_SUPPORTS_TLS_IN_TLS = hasattr(
-    asyncio.sslproto._SSLProtocolTransport,
-    "_start_tls_compatible",
-)
+ASYNCIO_SUPPORTS_TLS_IN_TLS = sys.version_info >= (3, 11)
 
 
 @pytest.fixture
@@ -54,7 +52,9 @@ def secure_proxy_url(tls_certificate_pem_path):
     This fixture also spawns that instance and tears it down after the test.
     """
     proxypy_args = [
-        "--threaded",  # needed for windows
+        # --threadless does not work on windows, see
+        # https://github.com/abhinavsingh/proxy.py/issues/492
+        "--threaded" if os.name == "nt" else "--threadless",
         "--num-workers",
         "1",  # the tests only send one query anyway
         "--hostname",

From 68a55773a1ff862d6a805aafaa5c825f8786954c Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Thu, 23 Nov 2023 06:01:23 -0600
Subject: [PATCH 30/30] preen

---
 tests/test_proxy_functional.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py
index 402a7615e53..bd3a174a893 100644
--- a/tests/test_proxy_functional.py
+++ b/tests/test_proxy_functional.py
@@ -1,9 +1,7 @@
 # type: ignore
 import asyncio
-import functools
 import os
 import pathlib
-import platform
 import ssl
 import sys
 from re import match as match_regex
@@ -18,7 +16,6 @@
 import aiohttp
 from aiohttp import web
 from aiohttp.client_exceptions import ClientConnectionError
-from aiohttp.helpers import PY_310
 
 pytestmark = [
     pytest.mark.filterwarnings(
@@ -32,16 +29,6 @@
 ]
 
 
-secure_proxy_xfail = functools.partial(
-    pytest.mark.xfail,
-    (PY_310 and platform.system() != "Darwin") or platform.system() == "Windows",
-    reason=(
-        "The secure proxy fixture does not seem to work "
-        "under Python 3.10 on Linux and any Python on Windows. "
-        "See https://github.com/abhinavsingh/proxy.py/issues/622."
-    ),
-)
-
 ASYNCIO_SUPPORTS_TLS_IN_TLS = sys.version_info >= (3, 11)