From 4d72dca6869072fb073621f8b752225e216a92d9 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 5 Apr 2024 19:23:48 +0100 Subject: [PATCH] Upgrade to llhttp 9.2.1 (#8292) Fixes #8291. --- CHANGES/8292.feature.rst | 1 + aiohttp/http_parser.py | 9 +++--- tests/test_http_parser.py | 65 +++++++++++++++++++++++++++++---------- vendor/llhttp | 2 +- 4 files changed, 55 insertions(+), 22 deletions(-) create mode 100644 CHANGES/8292.feature.rst diff --git a/CHANGES/8292.feature.rst b/CHANGES/8292.feature.rst new file mode 100644 index 00000000000..6ca82503143 --- /dev/null +++ b/CHANGES/8292.feature.rst @@ -0,0 +1 @@ +Upgraded to LLHTTP 9.2.1, and started rejecting obsolete line folding in Python parser to match -- by :user:`Dreamsorcerer`. diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 39798b604c7..c568c7574f4 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -123,12 +123,11 @@ class ChunkState(IntEnum): class HeadersParser: def __init__( - self, - max_line_size: int = 8190, - max_field_size: int = 8190, + self, max_line_size: int = 8190, max_field_size: int = 8190, lax: bool = False ) -> None: self.max_line_size = max_line_size self.max_field_size = max_field_size + self._lax = lax def parse_headers( self, lines: List[bytes] @@ -175,7 +174,7 @@ def parse_headers( line = lines[lines_idx] # consume continuation lines - continuation = line and line[0] in (32, 9) # (' ', '\t') + continuation = self._lax and line and line[0] in (32, 9) # (' ', '\t') # Deprecated: https://www.rfc-editor.org/rfc/rfc9112.html#name-obsolete-line-folding if continuation: @@ -268,7 +267,7 @@ def __init__( self._payload_parser: Optional[HttpPayloadParser] = None self._auto_decompress = auto_decompress self._limit = limit - self._headers_parser = HeadersParser(max_line_size, max_field_size) + self._headers_parser = HeadersParser(max_line_size, max_field_size, self.lax) @abc.abstractmethod def parse_message(self, lines: List[bytes]) -> _MsgT: diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index be258a457f1..fcfee480cf8 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -107,8 +107,7 @@ def test_c_parser_loaded(): def test_parse_headers(parser: Any) -> None: text = b"""GET /test HTTP/1.1\r -test: line\r - continue\r +test: a line\r test2: data\r \r """ @@ -116,13 +115,24 @@ def test_parse_headers(parser: Any) -> None: assert len(messages) == 1 msg = messages[0][0] - assert list(msg.headers.items()) == [("test", "line continue"), ("test2", "data")] - assert msg.raw_headers == ((b"test", b"line continue"), (b"test2", b"data")) + assert list(msg.headers.items()) == [("test", "a line"), ("test2", "data")] + assert msg.raw_headers == ((b"test", b"a line"), (b"test2", b"data")) assert not msg.should_close assert msg.compression is None assert not msg.upgrade +def test_reject_obsolete_line_folding(parser: Any) -> None: + text = b"""GET /test HTTP/1.1\r +test: line\r + Content-Length: 48\r +test2: data\r +\r +""" + with pytest.raises(http_exceptions.BadHttpMessage): + parser.feed_data(text) + + @pytest.mark.skipif(NO_EXTENSIONS, reason="Only tests C parser.") def test_invalid_character(loop: Any, protocol: Any, request: Any) -> None: parser = HttpRequestParserC( @@ -352,8 +362,8 @@ def test_parse_delayed(parser: Any) -> None: def test_headers_multi_feed(parser: Any) -> None: text1 = b"GET /test HTTP/1.1\r\n" - text2 = b"test: line\r" - text3 = b"\n continue\r\n\r\n" + text2 = b"test: line" + text3 = b" continue\r\n\r\n" messages, upgrade, tail = parser.feed_data(text1) assert len(messages) == 0 @@ -714,31 +724,30 @@ def test_max_header_value_size_under_limit(parser: Any) -> None: @pytest.mark.parametrize("size", [40965, 8191]) -def test_max_header_value_size_continuation(parser: Any, size: Any) -> None: +def test_max_header_value_size_continuation(response: Any, size: Any) -> None: name = b"T" * (size - 5) - text = b"GET /test HTTP/1.1\r\n" b"data: test\r\n " + name + b"\r\n\r\n" + text = b"HTTP/1.1 200 Ok\r\ndata: test\r\n " + name + b"\r\n\r\n" match = f"400, message:\n Got more than 8190 bytes \\({size}\\) when reading" with pytest.raises(http_exceptions.LineTooLong, match=match): - parser.feed_data(text) + response.feed_data(text) -def test_max_header_value_size_continuation_under_limit(parser: Any) -> None: +def test_max_header_value_size_continuation_under_limit(response: Any) -> None: value = b"A" * 8185 - text = b"GET /test HTTP/1.1\r\n" b"data: test\r\n " + value + b"\r\n\r\n" + text = b"HTTP/1.1 200 Ok\r\ndata: test\r\n " + value + b"\r\n\r\n" - messages, upgrade, tail = parser.feed_data(text) + messages, upgrade, tail = response.feed_data(text) msg = messages[0][0] - assert msg.method == "GET" - assert msg.path == "/test" + assert msg.code == 200 + assert msg.reason == "Ok" assert msg.version == (1, 1) assert msg.headers == CIMultiDict({"data": "test " + value.decode()}) assert msg.raw_headers == ((b"data", b"test " + value),) - assert not msg.should_close + # assert not msg.should_close # TODO: https://github.com/nodejs/llhttp/issues/354 assert msg.compression is None assert not msg.upgrade assert not msg.chunked - assert msg.url == URL("/test") def test_http_request_parser(parser: Any) -> None: @@ -992,6 +1001,30 @@ def test_http_response_parser_utf8_without_reason(response: Any) -> None: assert not tail +def test_http_response_parser_obs_line_folding(response: Any) -> None: + text = b"HTTP/1.1 200 Ok\r\ntest: line\r\n continue\r\n\r\n" + + messages, upgraded, tail = response.feed_data(text) + assert len(messages) == 1 + msg = messages[0][0] + + assert msg.version == (1, 1) + assert msg.code == 200 + assert msg.reason == "Ok" + assert msg.headers == CIMultiDict([("TEST", "line continue")]) + assert msg.raw_headers == ((b"test", b"line continue"),) + assert not upgraded + assert not tail + + +@pytest.mark.dev_mode +def test_http_response_parser_strict_obs_line_folding(response: Any) -> None: + text = b"HTTP/1.1 200 Ok\r\ntest: line\r\n continue\r\n\r\n" + + with pytest.raises(http_exceptions.BadHttpMessage): + response.feed_data(text) + + @pytest.mark.parametrize("size", [40962, 8191]) def test_http_response_parser_bad_status_line_too_long( response: Any, size: Any diff --git a/vendor/llhttp b/vendor/llhttp index 533845688d1..b0b279fb5a6 160000 --- a/vendor/llhttp +++ b/vendor/llhttp @@ -1 +1 @@ -Subproject commit 533845688d173561b9cba33269130401add38567 +Subproject commit b0b279fb5a617ab3bc2fc11c5f8bd937aac687c1