proxy mode for http proxies

CHANGELOG.md

@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Change the type of `Extensions` from `Mapping[Str, Any]` to `MutableMapping[Str, Any]`. (#762)
 - Handle HTTP/1.1 half-closed connections gracefully. (#641)
 - Drop Python 3.7 support. (#727)
+- Added `httpcore.ProxyMode` to make HTTP proxy mode configurable. You can now force proxy connection to be TUNNEL or FORWARD. 
 
 ## 0.17.3 (July 5th, 2023)
 

docs/proxies.md

@@ -51,6 +51,28 @@ proxy = httpcore.HTTPProxy(
 )
 ```
 
+## Proxy Connection Mode
+
+There are two types of HTTP proxy:
+
+1. Forwarding (HTTP [absolute-url](https://tools.ietf.org/html/rfc7230#section-5.3.2))
+2. Tunneling (HTTP [`CONNECT` method](https://tools.ietf.org/html/rfc7231#section-4.3.6))
+
+By default we forwarding plain http requests and tunneling https requests.
+You can change this behavior with `proxy_mode: httpcore.ProxyMode` parameter:
+```py
+import httpcore
+import base64
+
+proxy = httpcore.HTTPProxy(
+    proxy_url="http://127.0.0.1:8080/",
+    proxy_mode=httpcore.ProxyMode.TUNNEL,  # Forces HTTP requests also use Tunneling
+)
+```
+
+Note that `ProxyMode.FORWARD` will enable forwarding https requests, so they will be visible for proxy server.
+It means handling TLS stuffs like certificate validation would on proxy side.
+
 ## Proxy SSL
 
 The `httpcore` package also supports HTTPS proxies for http and https destinations.

httpcore/__init__.py

@@ -34,7 +34,7 @@
     WriteError,
     WriteTimeout,
 )
-from ._models import URL, Origin, Request, Response
+from ._models import URL, Origin, ProxyMode, Request, Response
 from ._ssl import default_ssl_context
 from ._sync import (
     ConnectionInterface,
@@ -75,8 +75,9 @@ def __init__(self, *args, **kwargs):  # type: ignore
     "request",
     "stream",
     # models
-    "Origin",
     "URL",
+    "Origin",
+    "ProxyMode",
     "Request",
     "Response",
     # async

httpcore/_async/http_proxy.py

@@ -8,6 +8,7 @@
 from .._models import (
     URL,
     Origin,
+    ProxyMode,
     Request,
     Response,
     enforce_bytes,
@@ -25,7 +26,6 @@
 HeadersAsSequence = Sequence[Tuple[Union[bytes, str], Union[bytes, str]]]
 HeadersAsMapping = Mapping[Union[bytes, str], Union[bytes, str]]
 
-
 logger = logging.getLogger("httpcore.proxy")
 
 
@@ -75,6 +75,7 @@ def __init__(
         uds: Optional[str] = None,
         network_backend: Optional[AsyncNetworkBackend] = None,
         socket_options: Optional[Iterable[SOCKET_OPTION]] = None,
+        proxy_mode: Optional[ProxyMode] = None,
     ) -> None:
         """
         A connection pool for making HTTP requests.
@@ -110,6 +111,7 @@ def __init__(
                 `AF_INET6` address (IPv6).
             uds: Path to a Unix Domain Socket to use instead of TCP sockets.
             network_backend: A backend instance to use for handling network I/O.
+            proxy_mode: Allow HTTP connection be tunnelable and HTTPS be forwardable.
         """
         super().__init__(
             ssl_context=ssl_context,
@@ -136,6 +138,7 @@ def __init__(
         self._ssl_context = ssl_context
         self._proxy_ssl_context = proxy_ssl_context
         self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers")
+        self._proxy_mode = proxy_mode
         if proxy_auth is not None:
             username = enforce_bytes(proxy_auth[0], name="proxy_auth")
             password = enforce_bytes(proxy_auth[1], name="proxy_auth")
@@ -145,7 +148,9 @@ def __init__(
             ] + self._proxy_headers
 
     def create_connection(self, origin: Origin) -> AsyncConnectionInterface:
-        if origin.scheme == b"http":
+        if (self._proxy_mode is ProxyMode.FORWARD) or (
+            self._proxy_mode is None and origin.scheme == b"http"
+        ):
             return AsyncForwardHTTPConnection(
                 proxy_origin=self._proxy_url.origin,
                 proxy_headers=self._proxy_headers,
@@ -298,31 +303,33 @@ async def handle_async_request(self, request: Request) -> Response:
                     raise ProxyError(msg)
 
                 stream = connect_response.extensions["network_stream"]
-
-                # Upgrade the stream to SSL
-                ssl_context = (
-                    default_ssl_context()
-                    if self._ssl_context is None
-                    else self._ssl_context
-                )
-                alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"]
-                ssl_context.set_alpn_protocols(alpn_protocols)
-
-                kwargs = {
-                    "ssl_context": ssl_context,
-                    "server_hostname": self._remote_origin.host.decode("ascii"),
-                    "timeout": timeout,
-                }
-                async with Trace("start_tls", logger, request, kwargs) as trace:
-                    stream = await stream.start_tls(**kwargs)
-                    trace.return_value = stream
-
-                # Determine if we should be using HTTP/1.1 or HTTP/2
-                ssl_object = stream.get_extra_info("ssl_object")
-                http2_negotiated = (
-                    ssl_object is not None
-                    and ssl_object.selected_alpn_protocol() == "h2"
-                )
+                http2_negotiated = False
+
+                if self._remote_origin.scheme == b"https":
+                    # Upgrade the stream to SSL
+                    ssl_context = (
+                        default_ssl_context()
+                        if self._ssl_context is None
+                        else self._ssl_context
+                    )
+                    alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"]
+                    ssl_context.set_alpn_protocols(alpn_protocols)
+
+                    kwargs = {
+                        "ssl_context": ssl_context,
+                        "server_hostname": self._remote_origin.host.decode("ascii"),
+                        "timeout": timeout,
+                    }
+                    async with Trace("start_tls", logger, request, kwargs) as trace:
+                        stream = await stream.start_tls(**kwargs)
+                        trace.return_value = stream
+
+                    # Determine if we should be using HTTP/1.1 or HTTP/2
+                    ssl_object = stream.get_extra_info("ssl_object")
+                    http2_negotiated = (
+                        ssl_object is not None
+                        and ssl_object.selected_alpn_protocol() == "h2"
+                    )
 
                 # Create the HTTP/1.1 or HTTP/2 connection
                 if http2_negotiated or (self._http2 and not self._http1):

httpcore/_models.py

@@ -1,3 +1,4 @@
+import enum
 from typing import (
     Any,
     AsyncIterable,
@@ -482,3 +483,8 @@ async def aclose(self) -> None:
             )
         if hasattr(self.stream, "aclose"):
             await self.stream.aclose()
+
+
+class ProxyMode(enum.IntEnum):
+    FORWARD = 1
+    TUNNEL = 2

httpcore/_sync/http_proxy.py

@@ -8,6 +8,7 @@
 from .._models import (
     URL,
     Origin,
+    ProxyMode,
     Request,
     Response,
     enforce_bytes,
@@ -25,7 +26,6 @@
 HeadersAsSequence = Sequence[Tuple[Union[bytes, str], Union[bytes, str]]]
 HeadersAsMapping = Mapping[Union[bytes, str], Union[bytes, str]]
 
-
 logger = logging.getLogger("httpcore.proxy")
 
 
@@ -75,6 +75,7 @@ def __init__(
         uds: Optional[str] = None,
         network_backend: Optional[NetworkBackend] = None,
         socket_options: Optional[Iterable[SOCKET_OPTION]] = None,
+        proxy_mode: Optional[ProxyMode] = None,
     ) -> None:
         """
         A connection pool for making HTTP requests.
@@ -110,6 +111,7 @@ def __init__(
                 `AF_INET6` address (IPv6).
             uds: Path to a Unix Domain Socket to use instead of TCP sockets.
             network_backend: A backend instance to use for handling network I/O.
+            proxy_mode: Allow HTTP connection be tunnelable and HTTPS be forwardable.
         """
         super().__init__(
             ssl_context=ssl_context,
@@ -136,6 +138,7 @@ def __init__(
         self._ssl_context = ssl_context
         self._proxy_ssl_context = proxy_ssl_context
         self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers")
+        self._proxy_mode = proxy_mode
         if proxy_auth is not None:
             username = enforce_bytes(proxy_auth[0], name="proxy_auth")
             password = enforce_bytes(proxy_auth[1], name="proxy_auth")
@@ -145,7 +148,9 @@ def __init__(
             ] + self._proxy_headers
 
     def create_connection(self, origin: Origin) -> ConnectionInterface:
-        if origin.scheme == b"http":
+        if (self._proxy_mode is ProxyMode.FORWARD) or (
+            self._proxy_mode is None and origin.scheme == b"http"
+        ):
             return ForwardHTTPConnection(
                 proxy_origin=self._proxy_url.origin,
                 proxy_headers=self._proxy_headers,
@@ -298,31 +303,33 @@ def handle_request(self, request: Request) -> Response:
                     raise ProxyError(msg)
 
                 stream = connect_response.extensions["network_stream"]
-
-                # Upgrade the stream to SSL
-                ssl_context = (
-                    default_ssl_context()
-                    if self._ssl_context is None
-                    else self._ssl_context
-                )
-                alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"]
-                ssl_context.set_alpn_protocols(alpn_protocols)
-
-                kwargs = {
-                    "ssl_context": ssl_context,
-                    "server_hostname": self._remote_origin.host.decode("ascii"),
-                    "timeout": timeout,
-                }
-                with Trace("start_tls", logger, request, kwargs) as trace:
-                    stream = stream.start_tls(**kwargs)
-                    trace.return_value = stream
-
-                # Determine if we should be using HTTP/1.1 or HTTP/2
-                ssl_object = stream.get_extra_info("ssl_object")
-                http2_negotiated = (
-                    ssl_object is not None
-                    and ssl_object.selected_alpn_protocol() == "h2"
-                )
+                http2_negotiated = False
+
+                if self._remote_origin.scheme == b"https":
+                    # Upgrade the stream to SSL
+                    ssl_context = (
+                        default_ssl_context()
+                        if self._ssl_context is None
+                        else self._ssl_context
+                    )
+                    alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"]
+                    ssl_context.set_alpn_protocols(alpn_protocols)
+
+                    kwargs = {
+                        "ssl_context": ssl_context,
+                        "server_hostname": self._remote_origin.host.decode("ascii"),
+                        "timeout": timeout,
+                    }
+                    with Trace("start_tls", logger, request, kwargs) as trace:
+                        stream = stream.start_tls(**kwargs)
+                        trace.return_value = stream
+
+                    # Determine if we should be using HTTP/1.1 or HTTP/2
+                    ssl_object = stream.get_extra_info("ssl_object")
+                    http2_negotiated = (
+                        ssl_object is not None
+                        and ssl_object.selected_alpn_protocol() == "h2"
+                    )
 
                 # Create the HTTP/1.1 or HTTP/2 connection
                 if http2_negotiated or (self._http2 and not self._http1):