gh-119451: Fix OOM vulnerability in http.client

Reading the whole body of the HTTP response could cause OOM if
the Content-Length value is too large even if the server does not send
a large amount of data. Now the HTTP client reads large data by chunks,
therefore the amount of consumed memory is proportional to the amount
of sent data.

Author: serhiy-storchaka

Lib/http/client.py

@@ -111,6 +111,11 @@
 _MAXLINE = 65536
 _MAXHEADERS = 100
 
+# Data larger than this will be read in chunks, to prevent extreme
+# overallocation.
+_SAFE_BUF_SIZE = 1 << 20
+
+
 # Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
 #
 # VCHAR          = %x21-7E
@@ -637,9 +642,14 @@ def _safe_read(self, amt):
         reading. If the bytes are truly not available (due to EOF), then the
         IncompleteRead exception can be used to detect the problem.
         """
-        data = self.fp.read(amt)
-        if len(data) < amt:
-            raise IncompleteRead(data, amt-len(data))
+        cursize = min(amt, _SAFE_BUF_SIZE)
+        data = self.fp.read(cursize)
+        while len(data) < amt:
+            if len(data) < cursize:
+                raise IncompleteRead(data, amt-len(data))
+            delta = min(cursize, amt - cursize)
+            data += self.fp.read(cursize)
+            cursize += delta
         return data
 
     def _safe_readinto(self, b):

Lib/test/test_httplib.py

@@ -1436,6 +1436,39 @@ def run_server():
         thread.join()
         self.assertEqual(result, b"proxied data\n")
 
+    def test_large_content_length(self):
+        serv = socket.create_server((HOST, 0))
+        self.addCleanup(serv.close)
+
+        def run_server():
+            while True:
+                [conn, address] = serv.accept()
+                with conn:
+                    conn.recv(1024)
+                    if not size:
+                        break
+                    body = b"HTTP/1.1 200 Ok\r\nContent-Length: %d\r\n\r\nText" % size
+                    conn.sendall(body)
+
+        thread = threading.Thread(target=run_server)
+        thread.start()
+        self.addCleanup(thread.join, 1.0)
+
+        conn = client.HTTPConnection(*serv.getsockname())
+        try:
+            for w in range(18, 65):
+                size = 1 << w
+                conn.request("GET", "/")
+                with conn.getresponse() as response:
+                    self.assertRaises(client.IncompleteRead, response.read)
+                conn.close()
+        finally:
+            conn.close()
+            size = 0
+            conn.request("GET", "/")
+            conn.close()
+            thread.join()
+
     def test_putrequest_override_domain_validation(self):
         """
         It should be possible to override the default validation

Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst

@@ -0,0 +1,3 @@
+Fix OOM vulnerability in :mod:`http.client`, when reading the whole body of
+a specially prepared small HTTP response could cause consuming an arbitrary
+amount of memory.