File CVE-2025-53643.patch of Package python-aiohttp.39906
From e8d774f635dc6d1cd3174d0e38891da5de0e2b6a Mon Sep 17 00:00:00 2001
From: Sam Bull <git@sambull.org>
Date: Wed, 9 Jul 2025 19:55:22 +0100
Subject: [PATCH] Add trailer parsing logic (#11269) (#11287)
(cherry picked from commit 7dd4b5535e6bf9c2d2f05fde638517bff065ba74)
---
aiohttp/http_parser.py | 70 +++++++++---------
aiohttp/multipart.py | 2 +-
tests/test_http_parser.py | 148 ++++++++++++++++----------------------
4 files changed, 100 insertions(+), 121 deletions(-)
create mode 100644 CHANGES/11269.feature.rst
Index: aiohttp-3.9.3/aiohttp/http_parser.py
===================================================================
--- aiohttp-3.9.3.orig/aiohttp/http_parser.py
+++ aiohttp-3.9.3/aiohttp/http_parser.py
@@ -139,8 +138,8 @@ class HeadersParser:
# note: "raw" does not mean inclusion of OWS before/after the field value
raw_headers = []
- lines_idx = 1
- line = lines[1]
+ lines_idx = 0
+ line = lines[lines_idx]
line_count = len(lines)
while line:
@@ -386,6 +383,7 @@ class HttpParser(abc.ABC, Generic[_MsgT]
response_with_body=self.response_with_body,
auto_decompress=self._auto_decompress,
lax=self.lax,
+ headers_parser=self._headers_parser,
)
if not payload_parser.done:
self._payload_parser = payload_parser
@@ -405,6 +403,7 @@ class HttpParser(abc.ABC, Generic[_MsgT]
readall=True,
auto_decompress=self._auto_decompress,
lax=self.lax,
+ headers_parser=self._headers_parser,
)
elif not empty_body and length is None and self.read_until_eof:
payload = StreamReader(
@@ -424,6 +423,7 @@ class HttpParser(abc.ABC, Generic[_MsgT]
response_with_body=self.response_with_body,
auto_decompress=self._auto_decompress,
lax=self.lax,
+ headers_parser=self._headers_parser,
)
if not payload_parser.done:
self._payload_parser = payload_parser
@@ -457,6 +457,8 @@ class HttpParser(abc.ABC, Generic[_MsgT]
eof = True
data = b""
+ if isinstance(exc, (InvalidHeader, TransferEncodingError)):
+ raise
if eof:
start_pos = 0
@@ -621,7 +623,7 @@ class HttpRequestParser(HttpParser[RawRe
compression,
upgrade,
chunked,
- ) = self.parse_headers(lines)
+ ) = self.parse_headers(lines[1:])
if close is None: # then the headers weren't set in the request
if version_o <= HttpVersion10: # HTTP 1.0 must asks to not close
@@ -701,7 +703,7 @@ class HttpResponseParser(HttpParser[RawR
compression,
upgrade,
chunked,
- ) = self.parse_headers(lines)
+ ) = self.parse_headers(lines[1:])
if close is None:
close = version_o <= HttpVersion10
@@ -732,6 +734,8 @@ class HttpPayloadParser:
response_with_body: bool = True,
auto_decompress: bool = True,
lax: bool = False,
+ *,
+ headers_parser: HeadersParser,
) -> None:
self._length = 0
self._type = ParseState.PARSE_NONE
@@ -740,6 +744,8 @@ class HttpPayloadParser:
self._chunk_tail = b""
self._auto_decompress = auto_decompress
self._lax = lax
+ self._headers_parser = headers_parser
+ self._trailer_lines: list[bytes] = []
self.done = False
# payload decompression wrapper
@@ -817,7 +823,6 @@ class HttpPayloadParser:
self._chunk_tail = b""
while chunk:
-
# read next chunk size
if self._chunk == ChunkState.PARSE_CHUNKED_SIZE:
pos = chunk.find(SEP)
@@ -827,7 +832,7 @@ class HttpPayloadParser:
size_b = chunk[:i] # strip chunk-extensions
# Verify no LF in the chunk-extension
if b"\n" in (ext := chunk[i:pos]):
- exc = BadHttpMessage(
+ exc = TransferEncodingError(
f"Unexpected LF in chunk-extension: {ext!r}"
)
set_exception(self.payload, exc)
@@ -848,7 +853,7 @@ class HttpPayloadParser:
chunk = chunk[pos + len(SEP) :]
if size == 0: # eof marker
- self._chunk = ChunkState.PARSE_MAYBE_TRAILERS
+ self._chunk = ChunkState.PARSE_TRAILERS
if self._lax and chunk.startswith(b"\r"):
chunk = chunk[1:]
else:
@@ -886,38 +891,31 @@ class HttpPayloadParser:
self._chunk_tail = chunk
return False, b""
- # if stream does not contain trailer, after 0\r\n
- # we should get another \r\n otherwise
- # trailers needs to be skipped until \r\n\r\n
- if self._chunk == ChunkState.PARSE_MAYBE_TRAILERS:
- head = chunk[: len(SEP)]
- if head == SEP:
- # end of stream
- self.payload.feed_eof()
- return True, chunk[len(SEP) :]
- # Both CR and LF, or only LF may not be received yet. It is
- # expected that CRLF or LF will be shown at the very first
- # byte next time, otherwise trailers should come. The last
- # CRLF which marks the end of response might not be
- # contained in the same TCP segment which delivered the
- # size indicator.
- if not head:
- return False, b""
- if head == SEP[:1]:
- self._chunk_tail = head
- return False, b""
- self._chunk = ChunkState.PARSE_TRAILERS
-
- # read and discard trailer up to the CRLF terminator
if self._chunk == ChunkState.PARSE_TRAILERS:
pos = chunk.find(SEP)
- if pos >= 0:
- chunk = chunk[pos + len(SEP) :]
- self._chunk = ChunkState.PARSE_MAYBE_TRAILERS
- else:
+ if pos < 0: # No line found
self._chunk_tail = chunk
return False, b""
+ line = chunk[:pos]
+ chunk = chunk[pos + len(SEP) :]
+ if SEP == b"\n": # For lax response parsing
+ line = line.rstrip(b"\r")
+ self._trailer_lines.append(line)
+
+ # \r\n\r\n found, end of stream
+ if self._trailer_lines[-1] == b"":
+ # Headers and trailers are defined the same way,
+ # so we reuse the HeadersParser here.
+ try:
+ trailers, raw_trailers = self._headers_parser.parse_headers(
+ self._trailer_lines
+ )
+ finally:
+ self._trailer_lines.clear()
+ self.payload.feed_eof()
+ return True, chunk
+
# Read all bytes until eof
elif self._type == ParseState.PARSE_UNTIL_EOF:
self.payload.feed_data(chunk, len(chunk))
Index: aiohttp-3.9.3/aiohttp/multipart.py
===================================================================
--- aiohttp-3.9.3.orig/aiohttp/multipart.py
+++ aiohttp-3.9.3/aiohttp/multipart.py
@@ -723,7 +723,7 @@ class MultipartReader:
raise ValueError(f"Invalid boundary {chunk!r}, expected {self._boundary!r}")
async def _read_headers(self) -> "CIMultiDictProxy[str]":
- lines = [b""]
+ lines = []
while True:
chunk = await self._content.readline()
chunk = chunk.strip()
Index: aiohttp-3.9.3/tests/test_http_parser.py
===================================================================
--- aiohttp-3.9.3.orig/tests/test_http_parser.py
+++ aiohttp-3.9.3/tests/test_http_parser.py
@@ -17,6 +17,7 @@ from aiohttp.base_protocol import BasePr
from aiohttp.http_parser import (
NO_EXTENSIONS,
DeflateBuffer,
+ HeadersParser,
HttpPayloadParser,
HttpRequestParser,
HttpRequestParserPy,
@@ -191,8 +192,8 @@ def test_bad_header_name(parser: Any, rf
(
"Content-Length: -5", # https://www.rfc-editor.org/rfc/rfc9110.html#name-content-length
"Content-Length: +256",
- "Content-Length: \N{superscript one}",
- "Content-Length: \N{mathematical double-struck digit one}",
+ "Content-Length: \N{SUPERSCRIPT ONE}",
+ "Content-Length: \N{MATHEMATICAL DOUBLE-STRUCK DIGIT ONE}",
"Foo: abc\rdef", # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.5-5
"Bar: abc\ndef",
"Baz: abc\x00def",
@@ -233,41 +234,13 @@ def test_content_length_transfer_encodin
parser.feed_data(text)
-def test_bad_chunked_py(loop: Any, protocol: Any) -> None:
+def test_bad_chunked(parser: HttpRequestParser) -> None:
"""Test that invalid chunked encoding doesn't allow content-length to be used."""
- parser = HttpRequestParserPy(
- protocol,
- loop,
- 2**16,
- max_line_size=8190,
- max_field_size=8190,
- )
text = (
b"GET / HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: chunked\r\n\r\n0_2e\r\n\r\n"
+ b"GET / HTTP/1.1\r\nHost: a\r\nContent-Length: 5\r\n\r\n0\r\n\r\n"
)
- messages, upgrade, tail = parser.feed_data(text)
- assert isinstance(messages[0][1].exception(), http_exceptions.TransferEncodingError)
-
-
-@pytest.mark.skipif(
- "HttpRequestParserC" not in dir(aiohttp.http_parser),
- reason="C based HTTP parser not available",
-)
-def test_bad_chunked_c(loop: Any, protocol: Any) -> None:
- """C parser behaves differently. Maybe we should align them later."""
- parser = HttpRequestParserC(
- protocol,
- loop,
- 2**16,
- max_line_size=8190,
- max_field_size=8190,
- )
- text = (
- b"GET / HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: chunked\r\n\r\n0_2e\r\n\r\n"
- + b"GET / HTTP/1.1\r\nHost: a\r\nContent-Length: 5\r\n\r\n0\r\n\r\n"
- )
- with pytest.raises(http_exceptions.BadHttpMessage):
+ with pytest.raises(http_exceptions.BadHttpMessage, match="0_2e"):
parser.feed_data(text)
@@ -422,49 +395,49 @@ def test_conn_default_1_1(parser) -> Non
def test_conn_close(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"connection: close\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\nconnection: close\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg = messages[0][0]
assert msg.should_close
def test_conn_close_1_0(parser) -> None:
- text = b"GET /test HTTP/1.0\r\n" b"connection: close\r\n\r\n"
+ text = b"GET /test HTTP/1.0\r\nconnection: close\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg = messages[0][0]
assert msg.should_close
def test_conn_keep_alive_1_0(parser) -> None:
- text = b"GET /test HTTP/1.0\r\n" b"connection: keep-alive\r\n\r\n"
+ text = b"GET /test HTTP/1.0\r\nconnection: keep-alive\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg = messages[0][0]
assert not msg.should_close
def test_conn_keep_alive_1_1(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"connection: keep-alive\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\nconnection: keep-alive\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg = messages[0][0]
assert not msg.should_close
def test_conn_other_1_0(parser) -> None:
- text = b"GET /test HTTP/1.0\r\n" b"connection: test\r\n\r\n"
+ text = b"GET /test HTTP/1.0\r\nconnection: test\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg = messages[0][0]
assert msg.should_close
def test_conn_other_1_1(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"connection: test\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\nconnection: test\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg = messages[0][0]
assert not msg.should_close
def test_request_chunked(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"transfer-encoding: chunked\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ntransfer-encoding: chunked\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg, payload = messages[0]
assert msg.chunked
@@ -486,7 +459,7 @@ def test_request_te_chunked_with_content
def test_request_te_chunked123(parser: Any) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"transfer-encoding: chunked123\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ntransfer-encoding: chunked123\r\n\r\n"
with pytest.raises(
http_exceptions.BadHttpMessage,
match="Request has invalid `Transfer-Encoding`",
@@ -495,11 +468,7 @@ def test_request_te_chunked123(parser: A
def test_conn_upgrade(parser: Any) -> None:
- text = (
- b"GET /test HTTP/1.1\r\n"
- b"connection: upgrade\r\n"
- b"upgrade: websocket\r\n\r\n"
- )
+ text = b"GET /test HTTP/1.1\r\nconnection: upgrade\r\nupgrade: websocket\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg = messages[0][0]
assert not msg.should_close
@@ -517,21 +486,21 @@ def test_bad_upgrade(parser) -> None:
def test_compression_empty(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"content-encoding: \r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ncontent-encoding: \r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg = messages[0][0]
assert msg.compression is None
def test_compression_deflate(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"content-encoding: deflate\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ncontent-encoding: deflate\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg = messages[0][0]
assert msg.compression == "deflate"
def test_compression_gzip(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"content-encoding: gzip\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ncontent-encoding: gzip\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg = messages[0][0]
assert msg.compression == "gzip"
@@ -539,21 +508,21 @@ def test_compression_gzip(parser) -> Non
@pytest.mark.skipif(brotli is None, reason="brotli is not installed")
def test_compression_brotli(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"content-encoding: br\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ncontent-encoding: br\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg = messages[0][0]
assert msg.compression == "br"
def test_compression_unknown(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"content-encoding: compress\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ncontent-encoding: compress\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg = messages[0][0]
assert msg.compression is None
def test_url_connect(parser: Any) -> None:
- text = b"CONNECT www.google.com HTTP/1.1\r\n" b"content-length: 0\r\n\r\n"
+ text = b"CONNECT www.google.com HTTP/1.1\r\ncontent-length: 0\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg, payload = messages[0]
assert upgrade
@@ -561,7 +530,7 @@ def test_url_connect(parser: Any) -> Non
def test_headers_connect(parser: Any) -> None:
- text = b"CONNECT www.google.com HTTP/1.1\r\n" b"content-length: 0\r\n\r\n"
+ text = b"CONNECT www.google.com HTTP/1.1\r\ncontent-length: 0\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg, payload = messages[0]
assert upgrade
@@ -570,8 +539,7 @@ def test_headers_connect(parser: Any) ->
def test_url_absolute(parser: Any) -> None:
text = (
- b"GET https://www.google.com/path/to.html HTTP/1.1\r\n"
- b"content-length: 0\r\n\r\n"
+ b"GET https://www.google.com/path/to.html HTTP/1.1\r\ncontent-length: 0\r\n\r\n"
)
messages, upgrade, tail = parser.feed_data(text)
msg, payload = messages[0]
@@ -581,21 +549,21 @@ def test_url_absolute(parser: Any) -> No
def test_headers_old_websocket_key1(parser: Any) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"SEC-WEBSOCKET-KEY1: line\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\nSEC-WEBSOCKET-KEY1: line\r\n\r\n"
with pytest.raises(http_exceptions.BadHttpMessage):
parser.feed_data(text)
def test_headers_content_length_err_1(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"content-length: line\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ncontent-length: line\r\n\r\n"
with pytest.raises(http_exceptions.BadHttpMessage):
parser.feed_data(text)
def test_headers_content_length_err_2(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"content-length: -1\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ncontent-length: -1\r\n\r\n"
with pytest.raises(http_exceptions.BadHttpMessage):
parser.feed_data(text)
@@ -618,7 +586,7 @@ _pad: Dict[bytes, str] = {
@pytest.mark.parametrize("pad2", _pad.keys(), ids=["post-" + n for n in _pad.values()])
@pytest.mark.parametrize("pad1", _pad.keys(), ids=["pre-" + n for n in _pad.values()])
def test_invalid_header_spacing(parser, pad1: bytes, pad2: bytes, hdr: bytes) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"%s%s%s: value\r\n\r\n" % (pad1, hdr, pad2)
+ text = b"GET /test HTTP/1.1\r\n%s%s%s: value\r\n\r\n" % (pad1, hdr, pad2)
expectation = pytest.raises(http_exceptions.BadHttpMessage)
if pad1 == pad2 == b"" and hdr != b"":
# one entry in param matrix is correct: non-empty name, not padded
@@ -633,19 +601,19 @@ def test_invalid_header_spacing(parser,
def test_empty_header_name(parser) -> None:
if not isinstance(response, HttpResponseParserPy):
pytest.xfail("Regression test for Py parser. May match C behaviour later.")
- text = b"GET /test HTTP/1.1\r\n" b":test\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\n:test\r\n\r\n"
with pytest.raises(http_exceptions.BadHttpMessage):
parser.feed_data(text)
def test_invalid_header(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"test line\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ntest line\r\n\r\n"
with pytest.raises(http_exceptions.BadHttpMessage):
parser.feed_data(text)
def test_invalid_name(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"test[]: line\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ntest[]: line\r\n\r\n"
with pytest.raises(http_exceptions.BadHttpMessage):
parser.feed_data(text)
@@ -682,7 +650,7 @@ def test_max_header_field_size_under_lim
@pytest.mark.parametrize("size", [40960, 8191])
def test_max_header_value_size(parser, size) -> None:
name = b"t" * size
- text = b"GET /test HTTP/1.1\r\n" b"data:" + name + b"\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ndata:" + name + b"\r\n\r\n"
match = f"400, message:\n Got more than 8190 bytes \\({size}\\) when reading"
with pytest.raises(http_exceptions.LineTooLong, match=match):
@@ -691,7 +659,7 @@ def test_max_header_value_size(parser, s
def test_max_header_value_size_under_limit(parser) -> None:
value = b"A" * 8190
- text = b"GET /test HTTP/1.1\r\n" b"data:" + value + b"\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ndata:" + value + b"\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg = messages[0][0]
@@ -710,7 +678,7 @@ def test_max_header_value_size_under_lim
@pytest.mark.parametrize("size", [40965, 8191])
def test_max_header_value_size_continuation(parser, size) -> None:
name = b"T" * (size - 5)
- text = b"GET /test HTTP/1.1\r\n" b"data: test\r\n " + name + b"\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ndata: test\r\n " + name + b"\r\n\r\n"
match = f"400, message:\n Got more than 8190 bytes \\({size}\\) when reading"
with pytest.raises(http_exceptions.LineTooLong, match=match):
@@ -719,7 +687,7 @@ def test_max_header_value_size_continuat
def test_max_header_value_size_continuation_under_limit(parser) -> None:
value = b"A" * 8185
- text = b"GET /test HTTP/1.1\r\n" b"data: test\r\n " + value + b"\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ndata: test\r\n " + value + b"\r\n\r\n"
messages, upgrade, tail = parser.feed_data(text)
msg = messages[0][0]
@@ -763,11 +731,11 @@ def test_http_request_bad_status_line(pa
_num: Dict[bytes, str] = {
# dangerous: accepted by Python int()
# unicodedata.category("\U0001D7D9") == 'Nd'
- "\N{mathematical double-struck digit one}".encode(): "utf8digit",
+ "\N{MATHEMATICAL DOUBLE-STRUCK DIGIT ONE}".encode(): "utf8digit",
# only added for interop tests, refused by Python int()
# unicodedata.category("\U000000B9") == 'No'
- "\N{superscript one}".encode(): "utf8number",
- "\N{superscript one}".encode("latin-1"): "latin1number",
+ "\N{SUPERSCRIPT ONE}".encode(): "utf8number",
+ "\N{SUPERSCRIPT ONE}".encode("latin-1"): "latin1number",
}
@@ -782,7 +750,7 @@ def test_http_request_bad_status_line_nu
def test_http_request_bad_status_line_separator(parser: Any) -> None:
# single code point, old, multibyte NFKC, multibyte NFKD
- utf8sep = "\N{arabic ligature sallallahou alayhe wasallam}".encode()
+ utf8sep = "\N{ARABIC LIGATURE SALLALLAHOU ALAYHE WASALLAM}".encode()
text = b"GET /ligature HTTP/1" + utf8sep + b"1\r\n\r\n"
with pytest.raises(http_exceptions.BadStatusLine):
parser.feed_data(text)
@@ -814,10 +782,11 @@ def test_http_request_parser_utf8_reques
pytest.xfail("Regression test for Py parser. May match C behaviour later.")
messages, upgrade, tail = parser.feed_data(
# note the truncated unicode sequence
- b"GET /P\xc3\xbcnktchen\xa0\xef\xb7 HTTP/1.1\r\n" +
+ b"GET /P\xc3\xbcnktchen\xa0\xef\xb7 HTTP/1.1\r\n"
+ +
# for easier grep: ASCII 0xA0 more commonly known as non-breaking space
# note the leading and trailing spaces
- "sTeP: \N{latin small letter sharp s}nek\t\N{no-break space} "
+ "sTeP: \N{LATIN SMALL LETTER SHARP S}nek\t\N{NO-BREAK SPACE} "
"\r\n\r\n".encode()
)
msg = messages[0][0]
@@ -1059,8 +1028,8 @@ async def test_http_response_parser_bad_
text = (
b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5 \r\nabcde\r\n0\r\n\r\n"
)
- messages, upgrade, tail = response.feed_data(text)
- assert isinstance(messages[0][1].exception(), http_exceptions.TransferEncodingError)
+ with pytest.raises(http_exceptions.TransferEncodingError, match="5"):
+ response.feed_data(text)
@pytest.mark.dev_mode
@@ -1110,7 +1079,7 @@ def test_http_response_parser_code_not_a
def test_http_request_chunked_payload(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"transfer-encoding: chunked\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ntransfer-encoding: chunked\r\n\r\n"
msg, payload = parser.feed_data(text)[0][0]
assert msg.chunked
@@ -1125,7 +1094,7 @@ def test_http_request_chunked_payload(pa
def test_http_request_chunked_payload_and_next_message(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"transfer-encoding: chunked\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ntransfer-encoding: chunked\r\n\r\n"
msg, payload = parser.feed_data(text)[0][0]
messages, upgraded, tail = parser.feed_data(
@@ -1147,7 +1116,7 @@ def test_http_request_chunked_payload_an
def test_http_request_chunked_payload_chunks(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"transfer-encoding: chunked\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ntransfer-encoding: chunked\r\n\r\n"
msg, payload = parser.feed_data(text)[0][0]
parser.feed_data(b"4\r\ndata\r")
@@ -1169,7 +1138,7 @@ def test_http_request_chunked_payload_ch
def test_parse_chunked_payload_chunk_extension(parser) -> None:
- text = b"GET /test HTTP/1.1\r\n" b"transfer-encoding: chunked\r\n\r\n"
+ text = b"GET /test HTTP/1.1\r\ntransfer-encoding: chunked\r\n\r\n"
msg, payload = parser.feed_data(text)[0][0]
parser.feed_data(b"4;test\r\ndata\r\n4\r\nline\r\n0\r\ntest: test\r\n\r\n")
@@ -1179,8 +1148,28 @@ def test_parse_chunked_payload_chunk_ext
assert payload.is_eof()
-def _test_parse_no_length_or_te_on_post(loop, protocol, request_cls):
- parser = request_cls(protocol, loop, readall=True)
+async def test_request_chunked_with_trailer(parser: HttpRequestParser) -> None:
+ text = b"GET /test HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n4\r\ntest\r\n0\r\ntest: trailer\r\nsecond: test trailer\r\n\r\n"
+ messages, upgraded, tail = parser.feed_data(text)
+ assert not tail
+ msg, payload = messages[0]
+ assert await payload.read() == b"test"
+
+ # TODO: Add assertion of trailers when API added.
+
+
+async def test_request_chunked_reject_bad_trailer(parser: HttpRequestParser) -> None:
+ text = b"GET /test HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n0\r\nbad\ntrailer\r\n\r\n"
+ with pytest.raises(http_exceptions.BadHttpMessage, match=r"b'bad\\ntrailer'"):
+ parser.feed_data(text)
+
+
+def test_parse_no_length_or_te_on_post(
+ loop: asyncio.AbstractEventLoop,
+ protocol: BaseProtocol,
+ request_cls: type[HttpRequestParser],
+) -> None:
+ parser = request_cls(protocol, loop, limit=2**16)
text = b"POST /test HTTP/1.1\r\n\r\n"
msg, payload = parser.feed_data(text)[0][0]
@@ -1189,14 +1178,14 @@ def _test_parse_no_length_or_te_on_post(
def test_parse_payload_response_without_body(loop, protocol, response_cls) -> None:
parser = response_cls(protocol, loop, 2**16, response_with_body=False)
- text = b"HTTP/1.1 200 Ok\r\n" b"content-length: 10\r\n\r\n"
+ text = b"HTTP/1.1 200 Ok\r\ncontent-length: 10\r\n\r\n"
msg, payload = parser.feed_data(text)[0][0]
assert payload.is_eof()
def test_parse_length_payload(response) -> None:
- text = b"HTTP/1.1 200 Ok\r\n" b"content-length: 4\r\n\r\n"
+ text = b"HTTP/1.1 200 Ok\r\ncontent-length: 4\r\n\r\n"
msg, payload = response.feed_data(text)[0][0]
assert not payload.is_eof()
@@ -1272,9 +1261,7 @@ def test_parse_content_length_than_chunk
assert b"first" == b"".join(d for d in payload._buffer)
text = (
- b"HTTP/1.1 200 OK\r\n"
- b"transfer-encoding: chunked\r\n\r\n"
- b"6\r\nsecond\r\n0\r\n\r\n"
+ b"HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n6\r\nsecond\r\n0\r\n\r\n"
)
msg, payload = response.feed_data(text)[0][0]
assert msg.version == HttpVersion(major=1, minor=1)
@@ -1317,9 +1304,7 @@ def test_parse_chunked_payload_empty_bod
assert payload.is_eof()
text = (
- b"HTTP/1.1 200 OK\r\n"
- b"transfer-encoding: chunked\r\n\r\n"
- b"6\r\nsecond\r\n0\r\n\r\n"
+ b"HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n6\r\nsecond\r\n0\r\n\r\n"
)
msg, payload = response.feed_data(text)[0][0]
assert msg.version == HttpVersion(major=1, minor=1)
@@ -1339,19 +1324,10 @@ def test_parse_chunked_payload_empty_bod
assert b"second" == b"".join(d for d in payload._buffer)
-@pytest.mark.skipif(NO_EXTENSIONS, reason="Only tests C parser.")
-async def test_parse_chunked_payload_with_lf_in_extensions_c_parser(
- loop: asyncio.AbstractEventLoop, protocol: BaseProtocol
+async def test_parse_chunked_payload_with_lf_in_extensions(
+ parser: HttpRequestParser,
) -> None:
- """Test the C-parser with a chunked payload that has a LF in the chunk extensions."""
- # The C parser will raise a BadHttpMessage from feed_data
- parser = HttpRequestParserC(
- protocol,
- loop,
- 2**16,
- max_line_size=8190,
- max_field_size=8190,
- )
+ """Test chunked payload that has a LF in the chunk extensions."""
payload = (
b"GET / HTTP/1.1\r\nHost: localhost:5001\r\n"
b"Transfer-Encoding: chunked\r\n\r\n2;\nxx\r\n4c\r\n0\r\n\r\n"
@@ -1362,31 +1338,6 @@ async def test_parse_chunked_payload_wit
parser.feed_data(payload)
-async def test_parse_chunked_payload_with_lf_in_extensions_py_parser(
- loop: asyncio.AbstractEventLoop, protocol: BaseProtocol
-) -> None:
- """Test the py-parser with a chunked payload that has a LF in the chunk extensions."""
- # The py parser will not raise the BadHttpMessage directly, but instead
- # it will set the exception on the StreamReader.
- parser = HttpRequestParserPy(
- protocol,
- loop,
- 2**16,
- max_line_size=8190,
- max_field_size=8190,
- )
- payload = (
- b"GET / HTTP/1.1\r\nHost: localhost:5001\r\n"
- b"Transfer-Encoding: chunked\r\n\r\n2;\nxx\r\n4c\r\n0\r\n\r\n"
- b"GET /admin HTTP/1.1\r\nHost: localhost:5001\r\n"
- b"Transfer-Encoding: chunked\r\n\r\n0\r\n\r\n"
- )
- messages, _, _ = parser.feed_data(payload)
- reader = messages[0][1]
- assert isinstance(reader.exception(), http_exceptions.BadHttpMessage)
- assert "\\nxx" in str(reader.exception())
-
-
def test_partial_url(parser: HttpRequestParser) -> None:
messages, upgrade, tail = parser.feed_data(b"GET /te")
assert len(messages) == 0
@@ -1472,10 +1423,8 @@ def test_parse_bad_method_for_c_parser_r
class TestParsePayload:
async def test_parse_eof_payload(self, stream) -> None:
- out = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
- )
- p = HttpPayloadParser(out, readall=True)
+ out = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
+ p = HttpPayloadParser(out, readall=True, headers_parser=HeadersParser())
p.feed_data(b"data")
p.feed_eof()
@@ -1483,37 +1432,31 @@ class TestParsePayload:
assert [(bytearray(b"data"), 4)] == list(out._buffer)
async def test_parse_no_body(self, stream) -> None:
- out = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
- )
- p = HttpPayloadParser(out, method="PUT")
+ out = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
+ p = HttpPayloadParser(out, method="PUT", headers_parser=HeadersParser())
assert out.is_eof()
assert p.done
async def test_parse_length_payload_eof(self, stream) -> None:
- out = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
- )
+ out = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
- p = HttpPayloadParser(out, length=4)
+ p = HttpPayloadParser(out, length=4, headers_parser=HeadersParser())
p.feed_data(b"da")
with pytest.raises(http_exceptions.ContentLengthError):
p.feed_eof()
async def test_parse_chunked_payload_size_error(self, stream) -> None:
- out = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
- )
- p = HttpPayloadParser(out, chunked=True)
+ out = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
+ p = HttpPayloadParser(out, chunked=True, headers_parser=HeadersParser())
with pytest.raises(http_exceptions.TransferEncodingError):
p.feed_data(b"blah\r\n")
assert isinstance(out.exception(), http_exceptions.TransferEncodingError)
async def test_parse_chunked_payload_split_end(self, protocol) -> None:
out = aiohttp.StreamReader(protocol, 2**16, loop=None)
- p = HttpPayloadParser(out, chunked=True)
+ p = HttpPayloadParser(out, chunked=True, headers_parser=HeadersParser())
p.feed_data(b"4\r\nasdf\r\n0\r\n")
p.feed_data(b"\r\n")
@@ -1522,7 +1465,7 @@ class TestParsePayload:
async def test_parse_chunked_payload_split_end2(self, protocol) -> None:
out = aiohttp.StreamReader(protocol, 2**16, loop=None)
- p = HttpPayloadParser(out, chunked=True)
+ p = HttpPayloadParser(out, chunked=True, headers_parser=HeadersParser())
p.feed_data(b"4\r\nasdf\r\n0\r\n\r")
p.feed_data(b"\n")
@@ -1531,7 +1474,7 @@ class TestParsePayload:
async def test_parse_chunked_payload_split_end_trailers(self, protocol) -> None:
out = aiohttp.StreamReader(protocol, 2**16, loop=None)
- p = HttpPayloadParser(out, chunked=True)
+ p = HttpPayloadParser(out, chunked=True, headers_parser=HeadersParser())
p.feed_data(b"4\r\nasdf\r\n0\r\n")
p.feed_data(b"Content-MD5: 912ec803b2ce49e4a541068d495ab570\r\n")
p.feed_data(b"\r\n")
@@ -1541,7 +1484,7 @@ class TestParsePayload:
async def test_parse_chunked_payload_split_end_trailers2(self, protocol) -> None:
out = aiohttp.StreamReader(protocol, 2**16, loop=None)
- p = HttpPayloadParser(out, chunked=True)
+ p = HttpPayloadParser(out, chunked=True, headers_parser=HeadersParser())
p.feed_data(b"4\r\nasdf\r\n0\r\n")
p.feed_data(b"Content-MD5: 912ec803b2ce49e4a541068d495ab570\r\n\r")
p.feed_data(b"\n")
@@ -1551,7 +1494,7 @@ class TestParsePayload:
async def test_parse_chunked_payload_split_end_trailers3(self, protocol) -> None:
out = aiohttp.StreamReader(protocol, 2**16, loop=None)
- p = HttpPayloadParser(out, chunked=True)
+ p = HttpPayloadParser(out, chunked=True, headers_parser=HeadersParser())
p.feed_data(b"4\r\nasdf\r\n0\r\nContent-MD5: ")
p.feed_data(b"912ec803b2ce49e4a541068d495ab570\r\n\r\n")
@@ -1560,18 +1503,16 @@ class TestParsePayload:
async def test_parse_chunked_payload_split_end_trailers4(self, protocol) -> None:
out = aiohttp.StreamReader(protocol, 2**16, loop=None)
- p = HttpPayloadParser(out, chunked=True)
- p.feed_data(b"4\r\nasdf\r\n0\r\n" b"C")
+ p = HttpPayloadParser(out, chunked=True, headers_parser=HeadersParser())
+ p.feed_data(b"4\r\nasdf\r\n0\r\nC")
p.feed_data(b"ontent-MD5: 912ec803b2ce49e4a541068d495ab570\r\n\r\n")
assert out.is_eof()
assert b"asdf" == b"".join(out._buffer)
async def test_http_payload_parser_length(self, stream) -> None:
- out = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
- )
- p = HttpPayloadParser(out, length=2)
+ out = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
+ p = HttpPayloadParser(out, length=2, headers_parser=HeadersParser())
eof, tail = p.feed_data(b"1245")
assert eof
@@ -1583,10 +1524,10 @@ class TestParsePayload:
COMPRESSED = b"x\x9cKI,I\x04\x00\x04\x00\x01\x9b"
length = len(COMPRESSED)
- out = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
+ out = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
+ p = HttpPayloadParser(
+ out, length=length, compression="deflate", headers_parser=HeadersParser()
)
- p = HttpPayloadParser(out, length=length, compression="deflate")
p.feed_data(COMPRESSED)
assert b"data" == b"".join(d for d, _ in out._buffer)
assert out.is_eof()
@@ -1597,10 +1538,10 @@ class TestParsePayload:
COMPRESSED = b"KI,I\x04\x00"
length = len(COMPRESSED)
- out = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
+ out = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
+ p = HttpPayloadParser(
+ out, length=length, compression="deflate", headers_parser=HeadersParser()
)
- p = HttpPayloadParser(out, length=length, compression="deflate")
p.feed_data(COMPRESSED)
assert b"data" == b"".join(d for d, _ in out._buffer)
assert out.is_eof()
@@ -1610,19 +1551,19 @@ class TestParsePayload:
COMPRESSED = b"\x18\x95KI,I\x04\x00\x04\x00\x01\x9b"
length = len(COMPRESSED)
- out = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
+ out = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
+ p = HttpPayloadParser(
+ out, length=length, compression="deflate", headers_parser=HeadersParser()
)
- p = HttpPayloadParser(out, length=length, compression="deflate")
p.feed_data(COMPRESSED)
assert b"data" == b"".join(d for d, _ in out._buffer)
assert out.is_eof()
async def test_http_payload_parser_deflate_split(self, stream) -> None:
- out = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
+ out = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
+ p = HttpPayloadParser(
+ out, compression="deflate", readall=True, headers_parser=HeadersParser()
)
- p = HttpPayloadParser(out, compression="deflate", readall=True)
# Feeding one correct byte should be enough to choose exact
# deflate decompressor
p.feed_data(b"x", 1)
@@ -1631,10 +1572,10 @@ class TestParsePayload:
assert b"data" == b"".join(d for d, _ in out._buffer)
async def test_http_payload_parser_deflate_split_err(self, stream) -> None:
- out = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
+ out = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
+ p = HttpPayloadParser(
+ out, compression="deflate", readall=True, headers_parser=HeadersParser()
)
- p = HttpPayloadParser(out, compression="deflate", readall=True)
# Feeding one wrong byte should be enough to choose exact
# deflate decompressor
p.feed_data(b"K", 1)
@@ -1643,20 +1584,21 @@ class TestParsePayload:
assert b"data" == b"".join(d for d, _ in out._buffer)
async def test_http_payload_parser_length_zero(self, stream) -> None:
- out = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
- )
- p = HttpPayloadParser(out, length=0)
+ out = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
+ p = HttpPayloadParser(out, length=0, headers_parser=HeadersParser())
assert p.done
assert out.is_eof()
@pytest.mark.skipif(brotli is None, reason="brotli is not installed")
async def test_http_payload_brotli(self, stream) -> None:
compressed = brotli.compress(b"brotli data")
- out = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
+ out = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
+ p = HttpPayloadParser(
+ out,
+ length=len(compressed),
+ compression="br",
+ headers_parser=HeadersParser(),
)
- p = HttpPayloadParser(out, length=len(compressed), compression="br")
p.feed_data(compressed)
assert b"brotli data" == b"".join(d for d, _ in out._buffer)
assert out.is_eof()
@@ -1664,9 +1606,7 @@ class TestParsePayload:
class TestDeflateBuffer:
async def test_feed_data(self, stream) -> None:
- buf = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
- )
+ buf = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
dbuf = DeflateBuffer(buf, "deflate")
dbuf.decompressor = mock.Mock()
@@ -1677,9 +1617,7 @@ class TestDeflateBuffer:
assert [b"line"] == list(d for d, _ in buf._buffer)
async def test_feed_data_err(self, stream) -> None:
- buf = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
- )
+ buf = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
dbuf = DeflateBuffer(buf, "deflate")
exc = ValueError()
@@ -1692,9 +1630,7 @@ class TestDeflateBuffer:
dbuf.feed_data(b"xsomedata", 9)
async def test_feed_eof(self, stream) -> None:
- buf = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
- )
+ buf = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
dbuf = DeflateBuffer(buf, "deflate")
dbuf.decompressor = mock.Mock()
@@ -1705,9 +1641,7 @@ class TestDeflateBuffer:
assert buf._eof
async def test_feed_eof_err_deflate(self, stream) -> None:
- buf = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
- )
+ buf = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
dbuf = DeflateBuffer(buf, "deflate")
dbuf.decompressor = mock.Mock()
@@ -1718,9 +1652,7 @@ class TestDeflateBuffer:
dbuf.feed_eof()
async def test_feed_eof_no_err_gzip(self, stream) -> None:
- buf = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
- )
+ buf = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
dbuf = DeflateBuffer(buf, "gzip")
dbuf.decompressor = mock.Mock()
@@ -1731,9 +1663,7 @@ class TestDeflateBuffer:
assert [b"line"] == list(d for d, _ in buf._buffer)
async def test_feed_eof_no_err_brotli(self, stream) -> None:
- buf = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
- )
+ buf = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
dbuf = DeflateBuffer(buf, "br")
dbuf.decompressor = mock.Mock()
@@ -1744,9 +1674,7 @@ class TestDeflateBuffer:
assert [b"line"] == list(d for d, _ in buf._buffer)
async def test_empty_body(self, stream) -> None:
- buf = aiohttp.FlowControlDataQueue(
- stream, 2**16, loop=asyncio.get_event_loop()
- )
+ buf = aiohttp.FlowControlDataQueue(stream, 2**16, loop=asyncio.get_event_loop())
dbuf = DeflateBuffer(buf, "deflate")
dbuf.feed_eof()
