Merge branch 'master' into fix-warning-display

.github/workflows/ci-cd.yml

@@ -1,6 +1,7 @@
 name: CI
 
 on:
+  merge_group:
   push:
     branches:
       - 'master'
@@ -50,7 +51,7 @@ jobs:
       with:
         python-version: 3.9
     - name: Cache PyPI
-      uses: actions/cache@v3.0.4
+      uses: actions/cache@v3.3.2
       with:
         key: pip-lint-${{ hashFiles('requirements/*.txt') }}
         path: ~/.cache/pip
@@ -111,14 +112,14 @@ jobs:
       with:
         submodules: true
     - name: Cache llhttp generated files
-      uses: actions/cache@v3.0.4
+      uses: actions/cache@v3.3.2
       id: cache
       with:
         key: llhttp-${{ hashFiles('vendor/llhttp/package.json', 'vendor/llhttp/src/**/*') }}
         path:  vendor/llhttp/build
     - name: Setup NodeJS
       if: steps.cache.outputs.cache-hit != 'true'
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: '14'
     - name: Generate llhttp sources
@@ -199,7 +200,7 @@ jobs:
       run: |
         echo "::set-output name=dir::$(pip cache dir)"    # - name: Cache
     - name: Cache PyPI
-      uses: actions/cache@v3.0.4
+      uses: actions/cache@v3.3.2
       with:
         key: pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}-${{ hashFiles('requirements/*.txt') }}
         path: ${{ steps.pip-cache.outputs.dir }}

CHANGES/7078.feature

@@ -0,0 +1 @@
+Added ``WebSocketResponse.get_extra_info()`` to access a protocol transport's extra info.

CHANGES/7306.bugfix

@@ -0,0 +1 @@
+Fixed ``ClientWebSocketResponse.close_code`` being erroneously set to ``None`` when there are concurrent async tasks receiving data and closing the connection.

CHANGES/7689.feature

@@ -0,0 +1 @@
+Allow ``link`` argument to be set to None/empty in HTTP 451 exception.

CHANGES/7700.bugfix

@@ -0,0 +1 @@
+Fix issue with insufficient HTTP method and version validation.

CHANGES/7712.bugfix

@@ -0,0 +1 @@
+Add check to validate that absolute URIs have schemes.

CHANGES/7715.bugfix

@@ -0,0 +1 @@
+Fix unhandled exception when Python HTTP parser encounters unpaired Unicode surrogates.

CHANGES/7719.bugfix

@@ -0,0 +1 @@
+Update parser to disallow invalid characters in header field names and stop accepting LF as a request line separator.

CHANGES/7733.doc

@@ -0,0 +1 @@
+Fix, update, and improve client exceptions documentation.

CONTRIBUTORS.txt

@@ -53,6 +53,7 @@ Arthur Darcet
 Austin Scola
 Ben Bader
 Ben Greiner
+Ben Kallus
 Ben Timby
 Benedikt Reinartz
 Bob Haddleton
@@ -144,6 +145,7 @@ Hrishikesh Paranjape
 Hu Bo
 Hugh Young
 Hugo Herter
+Hugo Hromic
 Hugo van Kemenade
 Hynek Schlawack
 Igor Alexandrov
@@ -317,6 +319,7 @@ Tolga Tezel
 Tomasz Trebski
 Toshiaki Tanaka
 Trinh Hoang Nhu
+Tymofii Tsiapa
 Vadim Suharnikov
 Vaibhav Sagar
 Vamsi Krishna Avula

README.rst

@@ -44,7 +44,7 @@ Key Features
 - Supports both client and server side of HTTP protocol.
 - Supports both client and server Web-Sockets out-of-the-box and avoids
   Callback Hell.
-- Provides Web-server with middlewares and plugable routing.
+- Provides Web-server with middleware and pluggable routing.
 
 
 Getting started

aiohttp/client_ws.py

@@ -191,7 +191,8 @@ async def send_json(
     async def close(self, *, code: int = WSCloseCode.OK, message: bytes = b"") -> bool:
         # we need to break `receive()` cycle first,
         # `close()` may be called from different task
-        if self._waiting is not None and not self._closed:
+        if self._waiting is not None and not self._closing:
+            self._closing = True
             self._reader.feed_data(WS_CLOSING_MESSAGE, 0)
             await self._waiting
 
@@ -210,7 +211,7 @@ async def close(self, *, code: int = WSCloseCode.OK, message: bytes = b"") -> bo
                 self._response.close()
                 return True
 
-            if self._closing:
+            if self._close_code:
                 self._response.close()
                 return True
 

aiohttp/http_exceptions.py

@@ -85,10 +85,9 @@ def __init__(
 
 class InvalidHeader(BadHttpMessage):
     def __init__(self, hdr: Union[bytes, str]) -> None:
-        if isinstance(hdr, bytes):
-            hdr = hdr.decode("utf-8", "surrogateescape")
-        super().__init__(f"Invalid HTTP Header: {hdr}")
-        self.hdr = hdr
+        hdr_s = hdr.decode(errors="backslashreplace") if isinstance(hdr, bytes) else hdr
+        super().__init__(f"Invalid HTTP header: {hdr!r}")
+        self.hdr = hdr_s
         self.args = (hdr,)
 
 

aiohttp/http_parser.py

@@ -34,6 +34,7 @@
     ContentEncodingError,
     ContentLengthError,
     InvalidHeader,
+    InvalidURLError,
     LineTooLong,
     TransferEncodingError,
 )
@@ -64,7 +65,9 @@
 #     token = 1*tchar
 METHRE: Final[Pattern[str]] = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
 VERSRE: Final[Pattern[str]] = re.compile(r"HTTP/(\d).(\d)")
-HDRRE: Final[Pattern[bytes]] = re.compile(rb"[\x00-\x1F\x7F()<>@,;:\[\]={} \t\"\\]")
+HDRRE: Final[Pattern[bytes]] = re.compile(
+    rb"[\x00-\x1F\x7F-\xFF()<>@,;:\[\]={} \t\"\\]"
+)
 HEXDIGIT = re.compile(rb"[0-9a-fA-F]+")
 
 
@@ -539,7 +542,7 @@ def parse_message(self, lines: List[bytes]) -> RawRequestMessage:
         # request line
         line = lines[0].decode("utf-8", "surrogateescape")
         try:
-            method, path, version = line.split(maxsplit=2)
+            method, path, version = line.split(" ", maxsplit=2)
         except ValueError:
             raise BadStatusLine(line) from None
 
@@ -549,11 +552,11 @@ def parse_message(self, lines: List[bytes]) -> RawRequestMessage:
             )
 
         # method
-        if not METHRE.match(method):
+        if not METHRE.fullmatch(method):
             raise BadStatusLine(method)
 
         # version
-        match = VERSRE.match(version)
+        match = VERSRE.fullmatch(version)
         if match is None:
             raise BadStatusLine(line)
         version_o = HttpVersion(int(match.group(1)), int(match.group(2)))
@@ -578,10 +581,18 @@ def parse_message(self, lines: List[bytes]) -> RawRequestMessage:
                 fragment=url_fragment,
                 encoded=True,
             )
+        elif path == "*" and method == "OPTIONS":
+            # asterisk-form,
+            url = URL(path, encoded=True)
         else:
             # absolute-form for proxy maybe,
             # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2
             url = URL(path, encoded=True)
+            if url.scheme == "":
+                # not absolute-form
+                raise InvalidURLError(
+                    path.encode(errors="surrogateescape").decode("latin1")
+                )
 
         # read headers
         (
@@ -652,7 +663,7 @@ def parse_message(self, lines: List[bytes]) -> RawResponseMessage:
             )
 
         # version
-        match = VERSRE.match(version)
+        match = VERSRE.fullmatch(version)
         if match is None:
             raise BadStatusLine(line)
         version_o = HttpVersion(int(match.group(1)), int(match.group(2)))
@@ -966,7 +977,7 @@ def end_http_chunk_receiving(self) -> None:
 
 try:
     if not NO_EXTENSIONS:
-        from ._http_parser import (  # type: ignore[import,no-redef]
+        from ._http_parser import (  # type: ignore[import-not-found,no-redef]
             HttpRequestParser,
             HttpResponseParser,
             RawRequestMessage,

aiohttp/http_websocket.py

@@ -160,7 +160,7 @@ def _websocket_mask_python(mask: bytes, data: bytearray) -> None:
     _websocket_mask = _websocket_mask_python
 else:
     try:
-        from ._websocket import _websocket_mask_cython  # type: ignore[import]
+        from ._websocket import _websocket_mask_cython  # type: ignore[import-not-found]
 
         _websocket_mask = _websocket_mask_cython
     except ImportError:  # pragma: no cover

aiohttp/http_writer.py

@@ -189,7 +189,7 @@ def _py_serialize_headers(status_line: str, headers: "CIMultiDict[str]") -> byte
 _serialize_headers = _py_serialize_headers
 
 try:
-    import aiohttp._http_writer as _http_writer  # type: ignore[import]
+    import aiohttp._http_writer as _http_writer  # type: ignore[import-not-found]
 
     _c_serialize_headers = _http_writer._serialize_headers
     if not NO_EXTENSIONS:

aiohttp/web_exceptions.py

@@ -424,7 +424,7 @@ class HTTPUnavailableForLegalReasons(HTTPClientError):
 
     def __init__(
         self,
-        link: StrOrURL,
+        link: Optional[StrOrURL],
         *,
         headers: Optional[LooseHeaders] = None,
         reason: Optional[str] = None,
@@ -434,11 +434,13 @@ def __init__(
         super().__init__(
             headers=headers, reason=reason, text=text, content_type=content_type
         )
-        self.headers["Link"] = f'<{str(link)}>; rel="blocked-by"'
-        self._link = URL(link)
+        self._link = None
+        if link:
+            self._link = URL(link)
+            self.headers["Link"] = f'<{str(self._link)}>; rel="blocked-by"'
 
     @property
-    def link(self) -> URL:
+    def link(self) -> Optional[URL]:
         return self._link
 
 

aiohttp/web_ws.py

@@ -323,6 +323,19 @@ def ws_protocol(self) -> Optional[str]:
     def compress(self) -> bool:
         return self._compress
 
+    def get_extra_info(self, name: str, default: Any = None) -> Any:
+        """Get optional transport information.
+
+        If no value associated with ``name`` is found, ``default`` is returned.
+        """
+        writer = self._writer
+        if writer is None:
+            return default
+        transport = writer.transport
+        if transport is None:
+            return default
+        return transport.get_extra_info(name, default)
+
     def exception(self) -> Optional[BaseException]:
         return self._exception