diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 3c0369951a7..276a76cb16f 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -260,7 +260,7 @@ jobs: needs: gen_llhttp runs-on: ubuntu-latest - timeout-minutes: 7 + timeout-minutes: 9 steps: - name: Checkout project uses: actions/checkout@v4 diff --git a/CHANGES.rst b/CHANGES.rst index 6f814e43817..a240be05961 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,34 @@ .. towncrier release notes start +3.11.18 (2025-04-20) +==================== + +Bug fixes +--------- + +- Disabled TLS in TLS warning (when using HTTPS proxies) for uvloop and newer Python versions -- by :user:`lezgomatt`. + + + *Related issues and pull requests on GitHub:* + :issue:`7686`. + + + +- Fixed reading fragmented WebSocket messages when the payload was masked -- by :user:`bdraco`. + + The problem first appeared in 3.11.17 + + + *Related issues and pull requests on GitHub:* + :issue:`10764`. + + + + +---- + + 3.11.17 (2025-04-19) ==================== diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 0866da52633..7c36b570e87 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -250,6 +250,7 @@ Martin Sucha Mathias Fröjdman Mathieu Dugré Matt VanEseltine +Matthew Go Matthias Marquardt Matthieu Hauglustaine Matthieu Rigal diff --git a/aiohttp/_websocket/reader_py.py b/aiohttp/_websocket/reader_py.py index df322a436dd..f022aa4d220 100644 --- a/aiohttp/_websocket/reader_py.py +++ b/aiohttp/_websocket/reader_py.py @@ -458,8 +458,7 @@ def _feed_data(self, data: bytes) -> None: self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) if self._has_mask: assert self._frame_mask is not None - payload_bytearray = bytearray() - payload_bytearray.join(self._payload_fragments) + payload_bytearray = bytearray(b"".join(self._payload_fragments)) websocket_mask(self._frame_mask, payload_bytearray) payload = payload_bytearray else: diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 08e6ae275ed..c525ed92191 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -1150,7 +1150,13 @@ def _warn_about_tls_in_tls( if req.request_info.url.scheme != "https": return - asyncio_supports_tls_in_tls = getattr( + # Check if uvloop is being used, which supports TLS in TLS, + # otherwise assume that asyncio's native transport is being used. + if type(underlying_transport).__module__.startswith("uvloop"): + return + + # Support in asyncio was added in Python 3.11 (bpo-44011) + asyncio_supports_tls_in_tls = sys.version_info >= (3, 11) or getattr( underlying_transport, "_start_tls_compatible", False, diff --git a/tests/conftest.py b/tests/conftest.py index 6ede2ba59fb..573d992e464 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,6 +32,13 @@ except ImportError: TRUSTME = False + +try: + import uvloop +except ImportError: + uvloop = None # type: ignore[assignment] + + pytest_plugins = ("aiohttp.pytest_plugin", "pytester") IS_HPUX = sys.platform.startswith("hp-ux") @@ -234,6 +241,16 @@ def selector_loop() -> Iterator[asyncio.AbstractEventLoop]: yield _loop +@pytest.fixture +def uvloop_loop() -> Iterator[asyncio.AbstractEventLoop]: + policy = uvloop.EventLoopPolicy() + asyncio.set_event_loop_policy(policy) + + with loop_context(policy.new_event_loop) as _loop: + asyncio.set_event_loop(_loop) + yield _loop + + @pytest.fixture def netrc_contents( tmp_path: Path, diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py index 256bff1f030..8ba37b248c6 100644 --- a/tests/test_proxy_functional.py +++ b/tests/test_proxy_functional.py @@ -1,6 +1,7 @@ import asyncio import os import pathlib +import platform import ssl import sys from re import match as match_regex @@ -240,6 +241,32 @@ async def test_https_proxy_unsupported_tls_in_tls( await asyncio.sleep(0.1) +@pytest.mark.usefixtures("uvloop_loop") +@pytest.mark.skipif( + platform.system() == "Windows" or sys.implementation.name != "cpython", + reason="uvloop is not supported on Windows and non-CPython implementations", +) +@pytest.mark.filterwarnings(r"ignore:.*ssl.OP_NO_SSL*") +# Filter out the warning from +# https://github.com/abhinavsingh/proxy.py/blob/30574fd0414005dfa8792a6e797023e862bdcf43/proxy/common/utils.py#L226 +# otherwise this test will fail because the proxy will die with an error. +async def test_uvloop_secure_https_proxy( + client_ssl_ctx: ssl.SSLContext, + secure_proxy_url: URL, +) -> None: + """Ensure HTTPS sites are accessible through a secure proxy without warning when using uvloop.""" + conn = aiohttp.TCPConnector() + sess = aiohttp.ClientSession(connector=conn) + url = URL("https://example.com") + + async with sess.get(url, proxy=secure_proxy_url, ssl=client_ssl_ctx) as response: + assert response.status == 200 + + await sess.close() + await conn.close() + await asyncio.sleep(0.1) + + @pytest.fixture def proxy_test_server( aiohttp_raw_server: AiohttpRawServer, diff --git a/tests/test_websocket_parser.py b/tests/test_websocket_parser.py index 6199abae359..808cac3380a 100644 --- a/tests/test_websocket_parser.py +++ b/tests/test_websocket_parser.py @@ -1,5 +1,6 @@ import asyncio import pickle +import random import struct from typing import Optional, Union from unittest import mock @@ -7,7 +8,14 @@ import pytest from aiohttp._websocket import helpers as _websocket_helpers -from aiohttp._websocket.helpers import PACK_CLOSE_CODE, PACK_LEN1, PACK_LEN2 +from aiohttp._websocket.helpers import ( + PACK_CLOSE_CODE, + PACK_LEN1, + PACK_LEN2, + PACK_LEN3, + PACK_RANDBITS, + websocket_mask, +) from aiohttp._websocket.models import WS_DEFLATE_TRAILING from aiohttp._websocket.reader import WebSocketDataQueue from aiohttp.base_protocol import BaseProtocol @@ -52,6 +60,7 @@ def build_frame( noheader: bool = False, is_fin: bool = True, ZLibBackend: Optional[ZLibBackendWrapper] = None, + mask: bool = False, ) -> bytes: # Send a frame over the websocket with message as its payload. compress = False @@ -72,11 +81,21 @@ def build_frame( if compress: header_first_byte |= 0x40 + mask_bit = 0x80 if mask else 0 + if msg_length < 126: - header = PACK_LEN1(header_first_byte, msg_length) + header = PACK_LEN1(header_first_byte, msg_length | mask_bit) + elif msg_length < 65536: + header = PACK_LEN2(header_first_byte, 126 | mask_bit, msg_length) else: - assert msg_length < (1 << 16) - header = PACK_LEN2(header_first_byte, 126, msg_length) + header = PACK_LEN3(header_first_byte, 127 | mask_bit, msg_length) + + if mask: + assert not noheader + mask_bytes = PACK_RANDBITS(random.getrandbits(32)) + message_arr = bytearray(message) + websocket_mask(mask_bytes, message_arr) + return header + mask_bytes + message_arr if noheader: return message @@ -352,6 +371,51 @@ def test_fragmentation_header( assert res == WSMessageText(data="a", size=1, extra="") +def test_large_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 131072 + data = build_frame(large_payload, WSMsgType.BINARY) + parser._feed_data(data) + + res = out._buffer[0] + assert res == WSMessageBinary(data=large_payload, size=131072, extra="") + + +def test_large_masked_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 131072 + data = build_frame(large_payload, WSMsgType.BINARY, mask=True) + parser._feed_data(data) + + res = out._buffer[0] + assert res == WSMessageBinary(data=large_payload, size=131072, extra="") + + +def test_fragmented_masked_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 100 + data = build_frame(large_payload, WSMsgType.BINARY, mask=True) + for i in range(len(data)): + parser._feed_data(data[i : i + 1]) + + res = out._buffer[0] + assert res == WSMessageBinary(data=large_payload, size=100, extra="") + + +def test_large_fragmented_masked_message( + out: WebSocketDataQueue, parser: PatchableWebSocketReader +) -> None: + large_payload = b"b" * 131072 + data = build_frame(large_payload, WSMsgType.BINARY, mask=True) + for i in range(0, len(data), 16384): + parser._feed_data(data[i : i + 16384]) + res = out._buffer[0] + assert res == WSMessageBinary(data=large_payload, size=131072, extra="") + + def test_continuation( out: WebSocketDataQueue, parser: PatchableWebSocketReader ) -> None: