Recover support for HTTP-to-HTTPS upgrade in proxy

aiohttp/connector.py

@@ -988,6 +988,50 @@ async def _wrap_create_connection(
         except OSError as exc:
             raise client_error(req.connection_key, exc) from exc
 
+    def _fail_on_no_start_tls(self, req: "ClientRequest") -> None:
+        """Raise a :py:exc:`RuntimeError` on missing ``start_tls()``.
+
+        One case is that :py:meth:`asyncio.loop.start_tls` is not yet
+        implemented under Python 3.6. It is necessary for TLS-in-TLS so
+        that it is possible to send HTTPS queries through HTTPS proxies.
+
+        This doesn't affect regular HTTP requests, though.
+        """
+        if not req.is_ssl():
+            return
+
+        proxy_url = req.proxy
+        assert proxy_url is not None
+        if proxy_url.scheme != "https":
+            return
+
+        self._check_loop_for_start_tls()
+
+    def _check_loop_for_start_tls(self) -> None:
+        try:
+            self._loop.start_tls
+        except AttributeError as attr_exc:
+            raise RuntimeError(
+                "An HTTPS request is being sent through an HTTPS proxy. "
+                "This needs support for TLS in TLS but it is not implemented "
+                "in your runtime for the stdlib asyncio.\n\n"
+                "Please upgrade to Python 3.7 or higher. For more details, "
+                "please see:\n"
+                "* https://bugs.python.org/issue37179\n"
+                "* https://github.com/python/cpython/pull/28073\n"
+                "* https://docs.aiohttp.org/en/stable/"
+                "client_advanced.html#proxy-support\n"
+                "* https://github.com/aio-libs/aiohttp/discussions/6044\n",
+            ) from attr_exc
+
+    def _loop_supports_start_tls(self) -> bool:
+        try:
+            self._check_loop_for_start_tls()
+        except RuntimeError:
+            return False
+        else:
+            return True
+
     def _warn_about_tls_in_tls(
         self,
         underlying_transport: asyncio.Transport,
@@ -1161,6 +1205,9 @@ def drop_exception(fut: "asyncio.Future[List[Dict[str, Any]]]") -> None:
     async def _create_proxy_connection(
         self, req: "ClientRequest", traces: List["Trace"], timeout: "ClientTimeout"
     ) -> Tuple[asyncio.BaseTransport, ResponseHandler]:
+        self._fail_on_no_start_tls(req)
+        runtime_has_start_tls = self._loop_supports_start_tls()
+
         headers = {}  # type: Dict[str, str]
         if req.proxy_headers is not None:
             headers = req.proxy_headers  # type: ignore[assignment]
@@ -1195,7 +1242,8 @@ async def _create_proxy_connection(
                 proxy_req.headers[hdrs.PROXY_AUTHORIZATION] = auth
 
         if req.is_ssl():
-            self._warn_about_tls_in_tls(transport, req)
+            if runtime_has_start_tls:
+                self._warn_about_tls_in_tls(transport, req)
 
             # For HTTPS requests over HTTP proxy
             # we must notify proxy to tunnel connection
@@ -1220,7 +1268,7 @@ async def _create_proxy_connection(
                 # read_until_eof=True will ensure the connection isn't closed
                 # once the response is received and processed allowing
                 # START_TLS to work on the connection below.
-                protocol.set_response_params(read_until_eof=True)
+                protocol.set_response_params(read_until_eof=runtime_has_start_tls)
                 resp = await proxy_resp.start(conn)
             except BaseException:
                 proxy_resp.close()
@@ -1241,12 +1289,35 @@ async def _create_proxy_connection(
                             message=message,
                             headers=resp.headers,
                         )
+                    if not runtime_has_start_tls:
+                        rawsock = transport.get_extra_info("socket", default=None)
+                        if rawsock is None:
+                            raise RuntimeError(
+                                "Transport does not expose socket instance"
+                            )
+                        # Duplicate the socket, so now we can close proxy transport
+                        rawsock = rawsock.dup()
                 except BaseException:
                     # It shouldn't be closed in `finally` because it's fed to
                     # `loop.start_tls()` and the docs say not to touch it after
                     # passing there.
                     transport.close()
                     raise
+                finally:
+                    if not runtime_has_start_tls:
+                        transport.close()
+
+                if not runtime_has_start_tls:
+                    # HTTP proxy with support for upgrade to HTTPS
+                    sslcontext = self._get_ssl_context(req)
+                    return await self._wrap_create_connection(
+                        self._factory,
+                        timeout=timeout,
+                        ssl=sslcontext,
+                        sock=rawsock,
+                        server_hostname=req.host,
+                        req=req,
+                    )
 
                 return await self._start_tls_connection(
                     # Access the old transport for the last time before it's

aiohttp/helpers.py

@@ -54,6 +54,9 @@
 
 __all__ = ("BasicAuth", "ChainMapProxy", "ETag")
 
+IS_MACOS = platform.system() == "Darwin"
+IS_WINDOWS = platform.system() == "Windows"
+
 PY_36 = sys.version_info >= (3, 6)
 PY_37 = sys.version_info >= (3, 7)
 PY_38 = sys.version_info >= (3, 8)
@@ -213,9 +216,7 @@ def netrc_from_env() -> Optional[netrc.netrc]:
             )
             return None
 
-        netrc_path = home_dir / (
-            "_netrc" if platform.system() == "Windows" else ".netrc"
-        )
+        netrc_path = home_dir / ("_netrc" if IS_WINDOWS else ".netrc")
 
     try:
         return netrc.netrc(str(netrc_path))

tests/test_proxy.py

@@ -5,11 +5,12 @@
 import unittest
 from unittest import mock
 
+import pytest
 from yarl import URL
 
 import aiohttp
 from aiohttp.client_reqrep import ClientRequest, ClientResponse
-from aiohttp.helpers import TimerNoop
+from aiohttp.helpers import PY_37, TimerNoop
 from aiohttp.test_utils import make_mocked_coro
 
 
@@ -351,6 +352,70 @@ async def make_conn():
                 connector._create_connection(req, None, aiohttp.ClientTimeout())
             )
 
+    @pytest.mark.skipif(
+        PY_37,
+        reason="The tested code path is only reachable below Python 3.7 because those "
+        "versions don't yet have `asyncio.loop.start_tls()` implemeneted",
+    )
+    @mock.patch("aiohttp.connector.ClientRequest")
+    def test_https_connect_runtime_error(self, ClientRequestMock) -> None:
+        proxy_req = ClientRequest(
+            "GET", URL("http://proxy.example.com"), loop=self.loop
+        )
+        ClientRequestMock.return_value = proxy_req
+
+        proxy_resp = ClientResponse(
+            "get",
+            URL("http://proxy.example.com"),
+            request_info=mock.Mock(),
+            writer=mock.Mock(),
+            continue100=None,
+            timer=TimerNoop(),
+            traces=[],
+            loop=self.loop,
+            session=mock.Mock(),
+        )
+        proxy_req.send = make_mocked_coro(proxy_resp)
+        proxy_resp.start = make_mocked_coro(mock.Mock(status=200))
+
+        async def make_conn():
+            return aiohttp.TCPConnector()
+
+        connector = self.loop.run_until_complete(make_conn())
+        connector._resolve_host = make_mocked_coro(
+            [
+                {
+                    "hostname": "hostname",
+                    "host": "127.0.0.1",
+                    "port": 80,
+                    "family": socket.AF_INET,
+                    "proto": 0,
+                    "flags": 0,
+                }
+            ]
+        )
+
+        tr, proto = mock.Mock(), mock.Mock()
+        tr.get_extra_info.return_value = None
+        self.loop.create_connection = make_mocked_coro((tr, proto))
+
+        req = ClientRequest(
+            "GET",
+            URL("https://www.python.org"),
+            proxy=URL("http://proxy.example.com"),
+            loop=self.loop,
+        )
+        with self.assertRaisesRegex(
+            RuntimeError, "Transport does not expose socket instance"
+        ):
+            self.loop.run_until_complete(
+                connector._create_connection(req, None, aiohttp.ClientTimeout())
+            )
+
+        self.loop.run_until_complete(proxy_req.close())
+        proxy_resp.close()
+        self.loop.run_until_complete(req.close())
+
     @mock.patch("aiohttp.connector.ClientRequest")
     def test_https_connect_http_proxy_error(self, ClientRequestMock) -> None:
         proxy_req = ClientRequest(

tests/test_proxy_functional.py

@@ -12,6 +12,7 @@
 import aiohttp
 from aiohttp import web
 from aiohttp.client_exceptions import ClientConnectionError
+from aiohttp.helpers import IS_MACOS, IS_WINDOWS, PY_37
 
 ASYNCIO_SUPPORTS_TLS_IN_TLS = hasattr(
     asyncio.sslproto._SSLProtocolTransport,
@@ -26,7 +27,6 @@ def secure_proxy_url(monkeypatch, tls_certificate_pem_path):
     This fixture also spawns that instance and tears it down after the test.
     """
     proxypy_args = [
-        "--threadless",  # use asyncio
         "--num-workers",
         "1",  # the tests only send one query anyway
         "--hostname",
@@ -38,6 +38,8 @@ def secure_proxy_url(monkeypatch, tls_certificate_pem_path):
         "--key-file",
         tls_certificate_pem_path,  # contains both key and cert
     ]
+    if not IS_MACOS and not IS_WINDOWS:
+        proxypy_args.append("--threadless")  # use asyncio
 
     class PatchedAccetorPool(proxy.core.acceptor.AcceptorPool):
         def listen(self):
@@ -111,7 +113,20 @@ def _pretend_asyncio_supports_tls_in_tls(
     )
 
 
-@pytest.mark.parametrize("web_server_endpoint_type", ("http", "https"))
+@pytest.mark.parametrize(
+    "web_server_endpoint_type",
+    (
+        "http",
+        pytest.param(
+            "https",
+            marks=pytest.mark.xfail(
+                not PY_37,
+                raises=RuntimeError,
+                reason="`asyncio.loop.start_tls()` is only implemeneted in Python 3.7",
+            ),
+        ),
+    ),
+)
 @pytest.mark.usefixtures("_pretend_asyncio_supports_tls_in_tls", "loop")
 async def test_secure_https_proxy_absolute_path(
     client_ssl_ctx,
@@ -137,6 +152,11 @@ async def test_secure_https_proxy_absolute_path(
     await conn.close()
 
 
+@pytest.mark.xfail(
+    not PY_37,
+    raises=RuntimeError,
+    reason="`asyncio.loop.start_tls()` is only implemeneted in Python 3.7",
+)
 @pytest.mark.parametrize("web_server_endpoint_type", ("https",))
 @pytest.mark.usefixtures("loop")
 async def test_https_proxy_unsupported_tls_in_tls(
@@ -197,6 +217,49 @@ async def test_https_proxy_unsupported_tls_in_tls(
     await conn.close()
 
 
+@pytest.mark.skipif(
+    PY_37,
+    reason="This test checks an error we emit below Python 3.7",
+)
+@pytest.mark.usefixtures("loop")
+async def test_https_proxy_missing_start_tls() -> None:
+    """Ensure error is raised for TLS-in-TLS w/ no ``start_tls()``."""
+    conn = aiohttp.TCPConnector()
+    sess = aiohttp.ClientSession(connector=conn)
+
+    expected_exception_reason = (
+        r"^"
+        r"An HTTPS request is being sent through an HTTPS proxy\. "
+        "This needs support for TLS in TLS but it is not implemented "
+        r"in your runtime for the stdlib asyncio\.\n\n"
+        r"Please upgrade to Python 3\.7 or higher\. For more details, "
+        r"please see:\n"
+        r"\* https://bugs\.python\.org/issue37179\n"
+        r"\* https://github\.com/python/cpython/pull/28073\n"
+        r"\* https://docs\.aiohttp\.org/en/stable/client_advanced\.html#proxy-support\n"
+        r"\* https://github\.com/aio-libs/aiohttp/discussions/6044\n"
+        r"$"
+    )
+
+    with pytest.raises(
+        RuntimeError,
+        match=expected_exception_reason,
+    ) as runtime_err:
+        await sess.get("https://python.org", proxy="https://proxy")
+
+    await sess.close()
+    await conn.close()
+
+    assert type(runtime_err.value.__cause__) is AttributeError
+
+    selector_event_loop_type = "Windows" if IS_WINDOWS else "Unix"
+    attr_err = (
+        f"^'_{selector_event_loop_type}SelectorEventLoop' object "
+        "has no attribute 'start_tls'$"
+    )
+    assert match_regex(attr_err, str(runtime_err.value.__cause__))
+
+
 @pytest.fixture
 def proxy_test_server(aiohttp_raw_server, loop, monkeypatch):
     # Handle all proxy requests and imitate remote server response.