From 83690e532065c33c569328d49b6e353f45587e8e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 19 Apr 2025 12:19:42 -1000 Subject: [PATCH 01/12] Increment version to 3.11.18.dev0 (#10758) --- aiohttp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index ab1d5bdedcc..e967a21bed0 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -1,4 +1,4 @@ -__version__ = "3.11.17" +__version__ = "3.11.18.dev0" from typing import TYPE_CHECKING, Tuple From e0cc020d59de9267146c55dbf082805213737653 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Apr 2025 20:55:00 -1000 Subject: [PATCH 02/12] Fix WebSocket reader with fragmented masked messages (#10764) --- CHANGES/10764.bugfix.rst | 3 ++ aiohttp/_websocket/reader_py.py | 3 +- tests/test_websocket_parser.py | 72 +++++++++++++++++++++++++++++++-- 3 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 CHANGES/10764.bugfix.rst diff --git a/CHANGES/10764.bugfix.rst b/CHANGES/10764.bugfix.rst new file mode 100644 index 00000000000..04cd71cd190 --- /dev/null +++ b/CHANGES/10764.bugfix.rst @@ -0,0 +1,3 @@ +Fixed reading fragmented WebSocket messages when the payload was masked -- by :user:`bdraco`. + +The problem first appeared in 3.11.17 diff --git a/aiohttp/_websocket/reader_py.py b/aiohttp/_websocket/reader_py.py index df322a436dd..f022aa4d220 100644 --- a/aiohttp/_websocket/reader_py.py +++ b/aiohttp/_websocket/reader_py.py @@ -458,8 +458,7 @@ def _feed_data(self, data: bytes) -> None: self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) if self._has_mask: assert self._frame_mask is not None - payload_bytearray = bytearray() - payload_bytearray.join(self._payload_fragments) + payload_bytearray = bytearray(b"".join(self._payload_fragments)) websocket_mask(self._frame_mask, payload_bytearray) payload = payload_bytearray else: diff --git a/tests/test_websocket_parser.py b/tests/test_websocket_parser.py index 6199abae359..808cac3380a 100644 --- a/tests/test_websocket_parser.py +++ b/tests/test_websocket_parser.py @@ -1,5 +1,6 @@ import asyncio import pickle +import random import struct from typing import Optional, Union from unittest import mock @@ -7,7 +8,14 @@ import pytest from aiohttp._websocket import helpers as _websocket_helpers -from aiohttp._websocket.helpers import PACK_CLOSE_CODE, PACK_LEN1, PACK_LEN2 +from aiohttp._websocket.helpers import ( + PACK_CLOSE_CODE, + PACK_LEN1, + PACK_LEN2, + PACK_LEN3, + PACK_RANDBITS, + websocket_mask, +) from aiohttp._websocket.models import WS_DEFLATE_TRAILING from aiohttp._websocket.reader import WebSocketDataQueue from aiohttp.base_protocol import BaseProtocol @@ -52,6 +60,7 @@ def build_frame( noheader: bool = False, is_fin: bool = True, ZLibBackend: Optional[ZLibBackendWrapper] = None, + mask: bool = False, ) -> bytes: # Send a frame over the websocket with message as its payload. compress = False @@ -72,11 +81,21 @@ def build_frame( if compress: header_first_byte |= 0x40 + mask_bit = 0x80 if mask else 0 + if msg_length < 126: - header = PACK_LEN1(header_first_byte, msg_length) + header = PACK_LEN1(header_first_byte, msg_length | mask_bit) + elif msg_length < 65536: + header = PACK_LEN2(header_first_byte, 126 | mask_bit, msg_length) else: - assert msg_length < (1 << 16) - header = PACK_LEN2(header_first_byte, 126, msg_length) + header = PACK_LEN3(header_first_byte, 127 | mask_bit, msg_length) + + if mask: + assert not noheader + mask_bytes = PACK_RANDBITS(random.getrandbits(32)) + message_arr = bytearray(message) + websocket_mask(mask_bytes, message_arr) + return header + mask_bytes + message_arr if noheader: return message @@ -352,6 +371,51 @@ def test_fragmentation_header( assert res == WSMessageText(data="a", size=1, extra="") +def test_large_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 131072 + data = build_frame(large_payload, WSMsgType.BINARY) + parser._feed_data(data) + + res = out._buffer[0] + assert res == WSMessageBinary(data=large_payload, size=131072, extra="") + + +def test_large_masked_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 131072 + data = build_frame(large_payload, WSMsgType.BINARY, mask=True) + parser._feed_data(data) + + res = out._buffer[0] + assert res == WSMessageBinary(data=large_payload, size=131072, extra="") + + +def test_fragmented_masked_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 100 + data = build_frame(large_payload, WSMsgType.BINARY, mask=True) + for i in range(len(data)): + parser._feed_data(data[i : i + 1]) + + res = out._buffer[0] + assert res == WSMessageBinary(data=large_payload, size=100, extra="") + + +def test_large_fragmented_masked_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 131072 + data = build_frame(large_payload, WSMsgType.BINARY, mask=True) + for i in range(0, len(data), 16384): + parser._feed_data(data[i : i + 16384]) + res = out._buffer[0] + assert res == WSMessageBinary(data=large_payload, size=131072, extra="") + + def test_continuation( out: WebSocketDataQueue, parser: PatchableWebSocketReader ) -> None: From feff48d43ae297254f8d3502839ddaeaacb8dad4 Mon Sep 17 00:00:00 2001 From: Matthew Go Date: Mon, 21 Apr 2025 15:58:22 +0800 Subject: [PATCH 03/12] Disable TLS in TLS warning for uvloop (#10726) --- CHANGES/7686.bugfix.rst | 1 + CONTRIBUTORS.txt | 1 + aiohttp/connector.py | 8 +++++++- tests/conftest.py | 17 +++++++++++++++++ tests/test_proxy_functional.py | 27 +++++++++++++++++++++++++++ 5 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 CHANGES/7686.bugfix.rst diff --git a/CHANGES/7686.bugfix.rst b/CHANGES/7686.bugfix.rst new file mode 100644 index 00000000000..7b575ff3564 --- /dev/null +++ b/CHANGES/7686.bugfix.rst @@ -0,0 +1 @@ +Disabled TLS in TLS warning (when using HTTPS proxies) for uvloop and newer Python versions -- by :user:`lezgomatt`. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 0866da52633..7c36b570e87 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -250,6 +250,7 @@ Martin Sucha Mathias Fröjdman Mathieu Dugré Matt VanEseltine +Matthew Go Matthias Marquardt Matthieu Hauglustaine Matthieu Rigal diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 08e6ae275ed..c525ed92191 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -1150,7 +1150,13 @@ def _warn_about_tls_in_tls( if req.request_info.url.scheme != "https": return - asyncio_supports_tls_in_tls = getattr( + # Check if uvloop is being used, which supports TLS in TLS, + # otherwise assume that asyncio's native transport is being used. + if type(underlying_transport).__module__.startswith("uvloop"): + return + + # Support in asyncio was added in Python 3.11 (bpo-44011) + asyncio_supports_tls_in_tls = sys.version_info >= (3, 11) or getattr( underlying_transport, "_start_tls_compatible", False, diff --git a/tests/conftest.py b/tests/conftest.py index 6ede2ba59fb..573d992e464 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,6 +32,13 @@ except ImportError: TRUSTME = False + +try: + import uvloop +except ImportError: + uvloop = None # type: ignore[assignment] + + pytest_plugins = ("aiohttp.pytest_plugin", "pytester") IS_HPUX = sys.platform.startswith("hp-ux") @@ -234,6 +241,16 @@ def selector_loop() -> Iterator[asyncio.AbstractEventLoop]: yield _loop +@pytest.fixture +def uvloop_loop() -> Iterator[asyncio.AbstractEventLoop]: + policy = uvloop.EventLoopPolicy() + asyncio.set_event_loop_policy(policy) + + with loop_context(policy.new_event_loop) as _loop: + asyncio.set_event_loop(_loop) + yield _loop + + @pytest.fixture def netrc_contents( tmp_path: Path, diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py index 256bff1f030..8ba37b248c6 100644 --- a/tests/test_proxy_functional.py +++ b/tests/test_proxy_functional.py @@ -1,6 +1,7 @@ import asyncio import os import pathlib +import platform import ssl import sys from re import match as match_regex @@ -240,6 +241,32 @@ async def test_https_proxy_unsupported_tls_in_tls( await asyncio.sleep(0.1) +@pytest.mark.usefixtures("uvloop_loop") +@pytest.mark.skipif( + platform.system() == "Windows" or sys.implementation.name != "cpython", + reason="uvloop is not supported on Windows and non-CPython implementations", +) +@pytest.mark.filterwarnings(r"ignore:.*ssl.OP_NO_SSL*") +# Filter out the warning from +# https://github.com/abhinavsingh/proxy.py/blob/30574fd0414005dfa8792a6e797023e862bdcf43/proxy/common/utils.py#L226 +# otherwise this test will fail because the proxy will die with an error. +async def test_uvloop_secure_https_proxy( + client_ssl_ctx: ssl.SSLContext, + secure_proxy_url: URL, +) -> None: + """Ensure HTTPS sites are accessible through a secure proxy without warning when using uvloop.""" + conn = aiohttp.TCPConnector() + sess = aiohttp.ClientSession(connector=conn) + url = URL("https://example.com") + + async with sess.get(url, proxy=secure_proxy_url, ssl=client_ssl_ctx) as response: + assert response.status == 200 + + await sess.close() + await conn.close() + await asyncio.sleep(0.1) + + @pytest.fixture def proxy_test_server( aiohttp_raw_server: AiohttpRawServer, From f69333ded38348c0db41d0dd07a2501958dceaff Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Apr 2025 22:07:35 -1000 Subject: [PATCH 04/12] [PR #10764/e0cc020 backport][3.11] Fix WebSocket reader with fragmented masked messages (#10765) --- CHANGES/10764.bugfix.rst | 3 +++ aiohttp/_websocket/reader_py.py | 3 +-- tests/test_websocket_parser.py | 45 +++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 CHANGES/10764.bugfix.rst diff --git a/CHANGES/10764.bugfix.rst b/CHANGES/10764.bugfix.rst new file mode 100644 index 00000000000..04cd71cd190 --- /dev/null +++ b/CHANGES/10764.bugfix.rst @@ -0,0 +1,3 @@ +Fixed reading fragmented WebSocket messages when the payload was masked -- by :user:`bdraco`. + +The problem first appeared in 3.11.17 diff --git a/aiohttp/_websocket/reader_py.py b/aiohttp/_websocket/reader_py.py index 2c7ae5779e2..f0060fd723c 100644 --- a/aiohttp/_websocket/reader_py.py +++ b/aiohttp/_websocket/reader_py.py @@ -439,8 +439,7 @@ def _feed_data(self, data: bytes) -> None: self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) if self._has_mask: assert self._frame_mask is not None - payload_bytearray = bytearray() - payload_bytearray.join(self._payload_fragments) + payload_bytearray = bytearray(b"".join(self._payload_fragments)) websocket_mask(self._frame_mask, payload_bytearray) payload = payload_bytearray else: diff --git a/tests/test_websocket_parser.py b/tests/test_websocket_parser.py index fc4888df5e5..d1d96f716fd 100644 --- a/tests/test_websocket_parser.py +++ b/tests/test_websocket_parser.py @@ -366,6 +366,51 @@ def test_fragmentation_header( assert res == (WSMessage(WSMsgType.TEXT, "a", ""), 1) +def test_large_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 131072 + data = build_frame(large_payload, WSMsgType.BINARY) + parser._feed_data(data) + + res = out._buffer[0] + assert res == ((WSMsgType.BINARY, large_payload, ""), 131072) + + +def test_large_masked_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 131072 + data = build_frame(large_payload, WSMsgType.BINARY, use_mask=True) + parser._feed_data(data) + + res = out._buffer[0] + assert res == ((WSMsgType.BINARY, large_payload, ""), 131072) + + +def test_fragmented_masked_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 100 + data = build_frame(large_payload, WSMsgType.BINARY, use_mask=True) + for i in range(len(data)): + parser._feed_data(data[i : i + 1]) + + res = out._buffer[0] + assert res == ((WSMsgType.BINARY, large_payload, ""), 100) + + +def test_large_fragmented_masked_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 131072 + data = build_frame(large_payload, WSMsgType.BINARY, use_mask=True) + for i in range(0, len(data), 16384): + parser._feed_data(data[i : i + 16384]) + res = out._buffer[0] + assert res == ((WSMsgType.BINARY, large_payload, ""), 131072) + + def test_continuation( out: WebSocketDataQueue, parser: PatchableWebSocketReader ) -> None: From 4182657c2cfd385c2d504f722e56c5cd144dd297 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Apr 2025 22:15:38 -1000 Subject: [PATCH 05/12] [PR #10764/e0cc020 backport][3.12] Fix WebSocket reader with fragmented masked messages (#10766) --- CHANGES/10764.bugfix.rst | 3 +++ aiohttp/_websocket/reader_py.py | 3 +-- tests/test_websocket_parser.py | 45 +++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 CHANGES/10764.bugfix.rst diff --git a/CHANGES/10764.bugfix.rst b/CHANGES/10764.bugfix.rst new file mode 100644 index 00000000000..04cd71cd190 --- /dev/null +++ b/CHANGES/10764.bugfix.rst @@ -0,0 +1,3 @@ +Fixed reading fragmented WebSocket messages when the payload was masked -- by :user:`bdraco`. + +The problem first appeared in 3.11.17 diff --git a/aiohttp/_websocket/reader_py.py b/aiohttp/_websocket/reader_py.py index 8a775742df1..855f9c6d600 100644 --- a/aiohttp/_websocket/reader_py.py +++ b/aiohttp/_websocket/reader_py.py @@ -447,8 +447,7 @@ def _feed_data(self, data: bytes) -> None: self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) if self._has_mask: assert self._frame_mask is not None - payload_bytearray = bytearray() - payload_bytearray.join(self._payload_fragments) + payload_bytearray = bytearray(b"".join(self._payload_fragments)) websocket_mask(self._frame_mask, payload_bytearray) payload = payload_bytearray else: diff --git a/tests/test_websocket_parser.py b/tests/test_websocket_parser.py index 52c34454886..37e15b64c18 100644 --- a/tests/test_websocket_parser.py +++ b/tests/test_websocket_parser.py @@ -368,6 +368,51 @@ def test_fragmentation_header( assert res == (WSMessage(WSMsgType.TEXT, "a", ""), 1) +def test_large_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 131072 + data = build_frame(large_payload, WSMsgType.BINARY) + parser._feed_data(data) + + res = out._buffer[0] + assert res == ((WSMsgType.BINARY, large_payload, ""), 131072) + + +def test_large_masked_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 131072 + data = build_frame(large_payload, WSMsgType.BINARY, use_mask=True) + parser._feed_data(data) + + res = out._buffer[0] + assert res == ((WSMsgType.BINARY, large_payload, ""), 131072) + + +def test_fragmented_masked_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 100 + data = build_frame(large_payload, WSMsgType.BINARY, use_mask=True) + for i in range(len(data)): + parser._feed_data(data[i : i + 1]) + + res = out._buffer[0] + assert res == ((WSMsgType.BINARY, large_payload, ""), 100) + + +def test_large_fragmented_masked_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 131072 + data = build_frame(large_payload, WSMsgType.BINARY, use_mask=True) + for i in range(0, len(data), 16384): + parser._feed_data(data[i : i + 16384]) + res = out._buffer[0] + assert res == ((WSMsgType.BINARY, large_payload, ""), 131072) + + def test_continuation( out: WebSocketDataQueue, parser: PatchableWebSocketReader ) -> None: From 51aa3589608bbbdb45fbad75f06b6380afe9df07 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Apr 2025 22:36:17 -1000 Subject: [PATCH 06/12] [PR #10726/feff48d backport][3.12] Disable TLS in TLS warning for uvloop (#10768) Co-authored-by: Matthew Go --- CHANGES/7686.bugfix.rst | 1 + CONTRIBUTORS.txt | 1 + aiohttp/connector.py | 8 +++++++- tests/conftest.py | 18 +++++++++++++++++- tests/test_proxy_functional.py | 27 +++++++++++++++++++++++++++ 5 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 CHANGES/7686.bugfix.rst diff --git a/CHANGES/7686.bugfix.rst b/CHANGES/7686.bugfix.rst new file mode 100644 index 00000000000..7b575ff3564 --- /dev/null +++ b/CHANGES/7686.bugfix.rst @@ -0,0 +1 @@ +Disabled TLS in TLS warning (when using HTTPS proxies) for uvloop and newer Python versions -- by :user:`lezgomatt`. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 4c44c5f4001..fb7bcf8e168 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -243,6 +243,7 @@ Martin Sucha Mathias Fröjdman Mathieu Dugré Matt VanEseltine +Matthew Go Matthias Marquardt Matthieu Hauglustaine Matthieu Rigal diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 2a41438ab6a..dd0d27a7054 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -1222,7 +1222,13 @@ def _warn_about_tls_in_tls( if req.request_info.url.scheme != "https": return - asyncio_supports_tls_in_tls = getattr( + # Check if uvloop is being used, which supports TLS in TLS, + # otherwise assume that asyncio's native transport is being used. + if type(underlying_transport).__module__.startswith("uvloop"): + return + + # Support in asyncio was added in Python 3.11 (bpo-44011) + asyncio_supports_tls_in_tls = sys.version_info >= (3, 11) or getattr( underlying_transport, "_start_tls_compatible", False, diff --git a/tests/conftest.py b/tests/conftest.py index be763400f45..437bccf9ba7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ from hashlib import md5, sha1, sha256 from pathlib import Path from tempfile import TemporaryDirectory -from typing import Any, Generator +from typing import Any, Generator, Iterator from unittest import mock from uuid import uuid4 @@ -32,6 +32,12 @@ except ImportError: TRUSTME = False + +try: + import uvloop +except ImportError: + uvloop = None # type: ignore[assignment] + pytest_plugins = ["aiohttp.pytest_plugin", "pytester"] IS_HPUX = sys.platform.startswith("hp-ux") @@ -227,6 +233,16 @@ def selector_loop(): yield _loop +@pytest.fixture +def uvloop_loop() -> Iterator[asyncio.AbstractEventLoop]: + policy = uvloop.EventLoopPolicy() + asyncio.set_event_loop_policy(policy) + + with loop_context(policy.new_event_loop) as _loop: + asyncio.set_event_loop(_loop) + yield _loop + + @pytest.fixture def netrc_contents( tmp_path: Path, diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py index 0921d5487bb..02d77700d96 100644 --- a/tests/test_proxy_functional.py +++ b/tests/test_proxy_functional.py @@ -1,6 +1,7 @@ import asyncio import os import pathlib +import platform import ssl import sys from re import match as match_regex @@ -202,6 +203,32 @@ async def test_https_proxy_unsupported_tls_in_tls( await asyncio.sleep(0.1) +@pytest.mark.usefixtures("uvloop_loop") +@pytest.mark.skipif( + platform.system() == "Windows" or sys.implementation.name != "cpython", + reason="uvloop is not supported on Windows and non-CPython implementations", +) +@pytest.mark.filterwarnings(r"ignore:.*ssl.OP_NO_SSL*") +# Filter out the warning from +# https://github.com/abhinavsingh/proxy.py/blob/30574fd0414005dfa8792a6e797023e862bdcf43/proxy/common/utils.py#L226 +# otherwise this test will fail because the proxy will die with an error. +async def test_uvloop_secure_https_proxy( + client_ssl_ctx: ssl.SSLContext, + secure_proxy_url: URL, +) -> None: + """Ensure HTTPS sites are accessible through a secure proxy without warning when using uvloop.""" + conn = aiohttp.TCPConnector() + sess = aiohttp.ClientSession(connector=conn) + url = URL("https://example.com") + + async with sess.get(url, proxy=secure_proxy_url, ssl=client_ssl_ctx) as response: + assert response.status == 200 + + await sess.close() + await conn.close() + await asyncio.sleep(0.1) + + @pytest.fixture def proxy_test_server(aiohttp_raw_server, loop, monkeypatch): # Handle all proxy requests and imitate remote server response. From a003df3c16f41d1ee29a844f6621fc573c4cefbc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Apr 2025 22:36:24 -1000 Subject: [PATCH 07/12] [PR #10726/feff48d backport][3.11] Disable TLS in TLS warning for uvloop (#10767) Co-authored-by: Matthew Go --- CHANGES/7686.bugfix.rst | 1 + CONTRIBUTORS.txt | 1 + aiohttp/connector.py | 8 +++++++- tests/conftest.py | 18 +++++++++++++++++- tests/test_proxy_functional.py | 27 +++++++++++++++++++++++++++ 5 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 CHANGES/7686.bugfix.rst diff --git a/CHANGES/7686.bugfix.rst b/CHANGES/7686.bugfix.rst new file mode 100644 index 00000000000..7b575ff3564 --- /dev/null +++ b/CHANGES/7686.bugfix.rst @@ -0,0 +1 @@ +Disabled TLS in TLS warning (when using HTTPS proxies) for uvloop and newer Python versions -- by :user:`lezgomatt`. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 953af52498a..9fec4933dc0 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -241,6 +241,7 @@ Martin Sucha Mathias Fröjdman Mathieu Dugré Matt VanEseltine +Matthew Go Matthias Marquardt Matthieu Hauglustaine Matthieu Rigal diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 7420bd6070a..7d5bcf755ec 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -1203,7 +1203,13 @@ def _warn_about_tls_in_tls( if req.request_info.url.scheme != "https": return - asyncio_supports_tls_in_tls = getattr( + # Check if uvloop is being used, which supports TLS in TLS, + # otherwise assume that asyncio's native transport is being used. + if type(underlying_transport).__module__.startswith("uvloop"): + return + + # Support in asyncio was added in Python 3.11 (bpo-44011) + asyncio_supports_tls_in_tls = sys.version_info >= (3, 11) or getattr( underlying_transport, "_start_tls_compatible", False, diff --git a/tests/conftest.py b/tests/conftest.py index 95a98cd4fc0..bceec5212a9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ from hashlib import md5, sha1, sha256 from pathlib import Path from tempfile import TemporaryDirectory -from typing import Any, Generator +from typing import Any, Generator, Iterator from unittest import mock from uuid import uuid4 @@ -27,6 +27,12 @@ except ImportError: TRUSTME = False + +try: + import uvloop +except ImportError: + uvloop = None # type: ignore[assignment] + pytest_plugins = ["aiohttp.pytest_plugin", "pytester"] IS_HPUX = sys.platform.startswith("hp-ux") @@ -193,6 +199,16 @@ def selector_loop(): yield _loop +@pytest.fixture +def uvloop_loop() -> Iterator[asyncio.AbstractEventLoop]: + policy = uvloop.EventLoopPolicy() + asyncio.set_event_loop_policy(policy) + + with loop_context(policy.new_event_loop) as _loop: + asyncio.set_event_loop(_loop) + yield _loop + + @pytest.fixture def netrc_contents( tmp_path: Path, diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py index 0921d5487bb..02d77700d96 100644 --- a/tests/test_proxy_functional.py +++ b/tests/test_proxy_functional.py @@ -1,6 +1,7 @@ import asyncio import os import pathlib +import platform import ssl import sys from re import match as match_regex @@ -202,6 +203,32 @@ async def test_https_proxy_unsupported_tls_in_tls( await asyncio.sleep(0.1) +@pytest.mark.usefixtures("uvloop_loop") +@pytest.mark.skipif( + platform.system() == "Windows" or sys.implementation.name != "cpython", + reason="uvloop is not supported on Windows and non-CPython implementations", +) +@pytest.mark.filterwarnings(r"ignore:.*ssl.OP_NO_SSL*") +# Filter out the warning from +# https://github.com/abhinavsingh/proxy.py/blob/30574fd0414005dfa8792a6e797023e862bdcf43/proxy/common/utils.py#L226 +# otherwise this test will fail because the proxy will die with an error. +async def test_uvloop_secure_https_proxy( + client_ssl_ctx: ssl.SSLContext, + secure_proxy_url: URL, +) -> None: + """Ensure HTTPS sites are accessible through a secure proxy without warning when using uvloop.""" + conn = aiohttp.TCPConnector() + sess = aiohttp.ClientSession(connector=conn) + url = URL("https://example.com") + + async with sess.get(url, proxy=secure_proxy_url, ssl=client_ssl_ctx) as response: + assert response.status == 200 + + await sess.close() + await conn.close() + await asyncio.sleep(0.1) + + @pytest.fixture def proxy_test_server(aiohttp_raw_server, loop, monkeypatch): # Handle all proxy requests and imitate remote server response. From a88a24360090ca8a98d9d4740825d9cf2fee2989 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Apr 2025 22:37:26 -1000 Subject: [PATCH 08/12] Increase benchmark timeout to 9 minutes in the CI (#10770) --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 3c0369951a7..276a76cb16f 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -260,7 +260,7 @@ jobs: needs: gen_llhttp runs-on: ubuntu-latest - timeout-minutes: 7 + timeout-minutes: 9 steps: - name: Checkout project uses: actions/checkout@v4 From 58b512cf07976651a5ef2beaff819d45bdced76a Mon Sep 17 00:00:00 2001 From: "patchback[bot]" <45432694+patchback[bot]@users.noreply.github.com> Date: Sun, 20 Apr 2025 22:37:46 -1000 Subject: [PATCH 09/12] [PR #10770/a88a2436 backport][3.11] Increase benchmark timeout to 9 minutes in the CI (#10771) Co-authored-by: J. Nick Koston --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index a794dc65d77..23266b2b2d5 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -244,7 +244,7 @@ jobs: needs: gen_llhttp runs-on: ubuntu-latest - timeout-minutes: 7 + timeout-minutes: 9 steps: - name: Checkout project uses: actions/checkout@v4 From 8b9974615b629d1059baca142c4312ea507c16b9 Mon Sep 17 00:00:00 2001 From: "patchback[bot]" <45432694+patchback[bot]@users.noreply.github.com> Date: Sun, 20 Apr 2025 22:37:58 -1000 Subject: [PATCH 10/12] [PR #10770/a88a2436 backport][3.12] Increase benchmark timeout to 9 minutes in the CI (#10772) Co-authored-by: J. Nick Koston --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index b00051b8668..ec85713319b 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -244,7 +244,7 @@ jobs: needs: gen_llhttp runs-on: ubuntu-latest - timeout-minutes: 7 + timeout-minutes: 9 steps: - name: Checkout project uses: actions/checkout@v4 From 0258e4de7d5c4e247e3ec7ae0cc672e438206c6d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Apr 2025 22:43:40 -1000 Subject: [PATCH 11/12] [PR #10761/d884799 backport][3.12] Speed up tests (#10769) Co-authored-by: Roman Postnov <59239573+dikos1337@users.noreply.github.com> --- CHANGES/9705.contrib.rst | 1 + CONTRIBUTORS.txt | 1 + setup.cfg | 1 + tests/conftest.py | 8 +++++++- tests/test_web_sendfile_functional.py | 11 +++++++++-- 5 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 CHANGES/9705.contrib.rst diff --git a/CHANGES/9705.contrib.rst b/CHANGES/9705.contrib.rst new file mode 100644 index 00000000000..771fb442629 --- /dev/null +++ b/CHANGES/9705.contrib.rst @@ -0,0 +1 @@ +Speed up tests by disabling ``blockbuster`` fixture for ``test_static_file_huge`` and ``test_static_file_huge_cancel`` tests -- by :user:`dikos1337`. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index fb7bcf8e168..3815ae6829d 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -297,6 +297,7 @@ Required Field Robert Lu Robert Nikolich Roman Podoliaka +Roman Postnov Rong Zhang Samir Akarioh Samuel Colvin diff --git a/setup.cfg b/setup.cfg index 9da34e0b5ce..83b33d01532 100644 --- a/setup.cfg +++ b/setup.cfg @@ -182,3 +182,4 @@ xfail_strict = true markers = dev_mode: mark test to run in dev mode. internal: tests which may cause issues for packagers, but should be run in aiohttp's CI. + skip_blockbuster: mark test to skip the blockbuster fixture. diff --git a/tests/conftest.py b/tests/conftest.py index 437bccf9ba7..de7f8316cb0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,7 +45,13 @@ @pytest.fixture(autouse=True) -def blockbuster(request): +def blockbuster(request: pytest.FixtureRequest) -> Iterator[None]: + # Allow selectively disabling blockbuster for specific tests + # using the @pytest.mark.skip_blockbuster marker. + if "skip_blockbuster" in request.node.keywords: + yield + return + # No blockbuster for benchmark tests. node = request.node.parent while node: diff --git a/tests/test_web_sendfile_functional.py b/tests/test_web_sendfile_functional.py index fc4db06a307..0c3e9ba68b5 100644 --- a/tests/test_web_sendfile_functional.py +++ b/tests/test_web_sendfile_functional.py @@ -11,6 +11,7 @@ import aiohttp from aiohttp import web from aiohttp.compression_utils import ZLibBackend +from aiohttp.pytest_plugin import AiohttpClient try: import brotlicffi as brotli @@ -642,7 +643,10 @@ async def test_static_file_directory_traversal_attack(aiohttp_client) -> None: await client.close() -async def test_static_file_huge(aiohttp_client, tmp_path) -> None: +@pytest.mark.skip_blockbuster +async def test_static_file_huge( + aiohttp_client: AiohttpClient, tmp_path: pathlib.Path +) -> None: file_path = tmp_path / "huge_data.unknown_mime_type" # fill 20MB file @@ -1073,7 +1077,10 @@ async def handler(request): await client.close() -async def test_static_file_huge_cancel(aiohttp_client, tmp_path) -> None: +@pytest.mark.skip_blockbuster +async def test_static_file_huge_cancel( + aiohttp_client: AiohttpClient, tmp_path: pathlib.Path +) -> None: file_path = tmp_path / "huge_data.unknown_mime_type" # fill 100MB file From 2be611a64cfcee84e96cb3a4e2fb69cfaa20e06b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Apr 2025 22:58:17 -1000 Subject: [PATCH 12/12] Release 3.11.18 (#10773) --- CHANGES.rst | 28 ++++++++++++++++++++++++++++ CHANGES/10764.bugfix.rst | 3 --- CHANGES/7686.bugfix.rst | 1 - aiohttp/__init__.py | 2 +- 4 files changed, 29 insertions(+), 5 deletions(-) delete mode 100644 CHANGES/10764.bugfix.rst delete mode 100644 CHANGES/7686.bugfix.rst diff --git a/CHANGES.rst b/CHANGES.rst index 3b62b221e4a..11fd19153e3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,34 @@ .. towncrier release notes start +3.11.18 (2025-04-20) +==================== + +Bug fixes +--------- + +- Disabled TLS in TLS warning (when using HTTPS proxies) for uvloop and newer Python versions -- by :user:`lezgomatt`. + + + *Related issues and pull requests on GitHub:* + :issue:`7686`. + + + +- Fixed reading fragmented WebSocket messages when the payload was masked -- by :user:`bdraco`. + + The problem first appeared in 3.11.17 + + + *Related issues and pull requests on GitHub:* + :issue:`10764`. + + + + +---- + + 3.11.17 (2025-04-19) ==================== diff --git a/CHANGES/10764.bugfix.rst b/CHANGES/10764.bugfix.rst deleted file mode 100644 index 04cd71cd190..00000000000 --- a/CHANGES/10764.bugfix.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed reading fragmented WebSocket messages when the payload was masked -- by :user:`bdraco`. - -The problem first appeared in 3.11.17 diff --git a/CHANGES/7686.bugfix.rst b/CHANGES/7686.bugfix.rst deleted file mode 100644 index 7b575ff3564..00000000000 --- a/CHANGES/7686.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Disabled TLS in TLS warning (when using HTTPS proxies) for uvloop and newer Python versions -- by :user:`lezgomatt`. diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index e967a21bed0..e3e0f3cc51e 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -1,4 +1,4 @@ -__version__ = "3.11.18.dev0" +__version__ = "3.11.18" from typing import TYPE_CHECKING, Tuple