diff --git a/system/HTTP/CURLRequest.php b/system/HTTP/CURLRequest.php index 1d60ca9e170f..6c3d118885ab 100644 --- a/system/HTTP/CURLRequest.php +++ b/system/HTTP/CURLRequest.php @@ -82,6 +82,27 @@ class CURLRequest extends OutgoingRequest ], ]; + /** + * Default values for when 'retry' is enabled. + * + * @var array> + */ + protected array $retryDefaults = [ + 'max_retries' => 3, + 'delay' => 1000, + 'max_delay' => 30_000, + 'status_codes' => [429, 503, 504], + 'curl_errors' => false, + 'respect_retry_after' => true, + ]; + + /** + * cURL error numbers that may succeed on another attempt. + * + * @var list + */ + protected array $transientCurlErrors = []; + /** * The number of milliseconds to delay before * sending the request. @@ -90,6 +111,11 @@ class CURLRequest extends OutgoingRequest */ protected $delay = 0.0; + /** + * The last cURL error number. + */ + protected int $lastCurlError = 0; + /** * The default options from the constructor. Applied to all requests. */ @@ -127,6 +153,14 @@ public function __construct(App $config, URI $uri, ?ResponseInterface $response throw HTTPException::forMissingCurl(); // @codeCoverageIgnore } + $this->transientCurlErrors = [ + CURLE_COULDNT_RESOLVE_HOST, + CURLE_COULDNT_CONNECT, + CURLE_OPERATION_TIMEDOUT, + CURLE_SEND_ERROR, + CURLE_RECV_ERROR, + ]; + parent::__construct(Method::GET, $uri); $this->responseOrig = $response ?? new Response(); @@ -374,6 +408,8 @@ public function send(string $method, string $url) { // Reset our curl options so we're on a fresh slate. $curlOptions = []; + $config = $this->config; + $retry = $this->normalizeRetryOption($config['retry'] ?? false); if (! empty($this->config['query']) && is_array($this->config['query'])) { // This is likely too naive a solution. @@ -394,14 +430,75 @@ public function send(string $method, string $url) // Disable @file uploads in post data. $curlOptions[CURLOPT_SAFE_UPLOAD] = true; - $curlOptions = $this->setCURLOptions($curlOptions, $this->config); + $curlOptions = $this->setCURLOptions($curlOptions, $config); $curlOptions = $this->applyMethod($method, $curlOptions); $curlOptions = $this->applyRequestHeaders($curlOptions); + if ($retry !== null) { + $curlOptions[CURLOPT_FAILONERROR] = false; + } + // Do we need to delay this request? if ($this->delay > 0) { - usleep((int) $this->delay * 1_000_000); + $this->sleep($this->delay); + } + + if ($retry === null) { + return $this->sendAttempt($curlOptions); + } + + $httpErrors = array_key_exists('http_errors', $config) ? (bool) $config['http_errors'] : true; + + return $this->sendWithRetries($curlOptions, $retry, $httpErrors); + } + + /** + * Sends the request until it succeeds or retry attempts are exhausted. + * + * @param array $curlOptions + * @param array> $retry + */ + protected function sendWithRetries(array $curlOptions, array $retry, bool $httpErrors): ResponseInterface + { + $attempt = 0; + + while (true) { + $this->response = clone $this->responseOrig; + + try { + $response = $this->sendAttempt($curlOptions); + } catch (HTTPException $e) { + if (! $this->shouldRetryCurlError($retry, $attempt)) { + throw $e; + } + + $this->sleep($this->getRetryDelay($retry, $attempt) / 1000); + $attempt++; + + continue; + } + + if (! $this->shouldRetryResponse($response, $retry, $attempt)) { + if ($httpErrors && $response->getStatusCode() >= 400) { + throw HTTPException::forCurlError((string) CURLE_HTTP_RETURNED_ERROR, 'The requested URL returned error: ' . $response->getStatusCode()); + } + + return $response; + } + + $this->sleep($this->getRetryDelay($retry, $attempt, $response) / 1000); + $attempt++; } + } + + /** + * Sends a single cURL request attempt and populates the response. + * + * @param array $curlOptions + */ + protected function sendAttempt(array $curlOptions): ResponseInterface + { + $this->lastCurlError = 0; $output = $this->sendRequest($curlOptions); @@ -430,6 +527,158 @@ public function send(string $method, string $url) return $this->response; } + /** + * Normalizes the retry option into retry settings. + * + * @return array>|null + */ + protected function normalizeRetryOption(mixed $retry): ?array + { + if (in_array($retry, [false, null, 0], true)) { + return null; + } + + $config = $this->retryDefaults; + + if (is_int($retry)) { + $config['max_retries'] = $retry; + } elseif (is_array($retry)) { + $config = array_merge($config, $retry); + } else { + return null; + } + + $config['max_retries'] = max(0, (int) $config['max_retries']); + + if ($config['max_retries'] === 0) { + return null; + } + + $config['delay'] = $this->normalizeRetryDelay($config['delay']); + $config['max_delay'] = max(0, (int) $config['max_delay']); + $config['status_codes'] = array_map(intval(...), (array) $config['status_codes']); + $config['curl_errors'] = (bool) $config['curl_errors']; + $config['respect_retry_after'] = (bool) $config['respect_retry_after']; + + return $config; + } + + /** + * Normalizes the retry delay setting. + * + * @return int|list + */ + protected function normalizeRetryDelay(mixed $delay): array|int + { + if (is_array($delay)) { + return array_map(static fn ($value): int => max(0, (int) $value), $delay); + } + + return max(0, (int) $delay); + } + + /** + * Determines whether a response should be retried. + * + * @param array> $retry + */ + protected function shouldRetryResponse(ResponseInterface $response, array $retry, int $attempt): bool + { + if ($attempt >= $retry['max_retries']) { + return false; + } + + return in_array($response->getStatusCode(), $retry['status_codes'], true); + } + + /** + * Determines whether a cURL error should be retried. + * + * @param array> $retry + */ + protected function shouldRetryCurlError(array $retry, int $attempt): bool + { + if ($attempt >= $retry['max_retries'] || $retry['curl_errors'] === false) { + return false; + } + + return in_array($this->lastCurlError, $this->transientCurlErrors, true); + } + + /** + * Returns the delay before the next retry attempt. + * + * @param array> $retry + */ + protected function getRetryDelay(array $retry, int $attempt, ?ResponseInterface $response = null): int + { + if ($response instanceof ResponseInterface && $retry['respect_retry_after'] === true) { + $retryAfter = $this->getRetryAfterDelay($response); + + if ($retryAfter !== null) { + return $this->limitRetryDelay($retryAfter * 1000, $retry); + } + } + + $delay = $retry['delay']; + + if (is_array($delay)) { + $lastDelay = $delay[array_key_last($delay)] ?? 0; + + return $this->limitRetryDelay((int) ($delay[$attempt] ?? $lastDelay), $retry); + } + + return $this->limitRetryDelay((int) $delay, $retry); + } + + /** + * Caps the retry delay when configured. + * + * @param array> $retry + */ + protected function limitRetryDelay(int $delay, array $retry): int + { + $maxDelay = (int) $retry['max_delay']; + + if ($maxDelay === 0) { + return $delay; + } + + return min($delay, $maxDelay); + } + + /** + * Returns the delay from a Retry-After header in seconds. + */ + protected function getRetryAfterDelay(ResponseInterface $response): ?int + { + $retryAfter = $response->getHeaderLine('Retry-After'); + + if ($retryAfter === '') { + return null; + } + + if (ctype_digit($retryAfter)) { + return (int) $retryAfter; + } + + $timestamp = strtotime($retryAfter); + + if ($timestamp === false) { + return null; + } + + return max(0, $timestamp - time()); + } + + /** + * Sleeps for the configured number of seconds. + */ + protected function sleep(float $seconds): void + { + usleep((int) ($seconds * 1_000_000)); + } + /** * Adds $this->headers to the cURL request. */ @@ -731,7 +980,9 @@ protected function sendRequest(array $curlOptions = []): string $output = curl_exec($ch); if ($output === false) { - throw HTTPException::forCurlError((string) curl_errno($ch), curl_error($ch)); + $this->lastCurlError = curl_errno($ch); + + throw HTTPException::forCurlError((string) $this->lastCurlError, curl_error($ch)); } return $output; diff --git a/system/Test/Mock/MockCURLRequest.php b/system/Test/Mock/MockCURLRequest.php index 059b83114927..c525db589649 100644 --- a/system/Test/Mock/MockCURLRequest.php +++ b/system/Test/Mock/MockCURLRequest.php @@ -14,6 +14,7 @@ namespace CodeIgniter\Test\Mock; use CodeIgniter\HTTP\CURLRequest; +use CodeIgniter\HTTP\Exceptions\HTTPException; use CodeIgniter\HTTP\URI; /** @@ -33,6 +34,21 @@ class MockCURLRequest extends CURLRequest */ protected $output = ''; + /** + * @var list + */ + protected array $outputs = []; + + /** + * @var list + */ + protected array $curlErrors = []; + + /** + * @var list + */ + protected array $sleeps = []; + /** * @param string $output * @@ -45,6 +61,30 @@ public function setOutput($output) return $this; } + /** + * @param list $outputs + * + * @return $this + */ + public function setOutputs(array $outputs) + { + $this->outputs = $outputs; + + return $this; + } + + /** + * @param list $curlErrors + * + * @return $this + */ + public function setCurlErrors(array $curlErrors) + { + $this->curlErrors = $curlErrors; + + return $this; + } + /** * @param array $curlOptions */ @@ -54,7 +94,28 @@ protected function sendRequest(array $curlOptions = []): string $this->curl_options = $curlOptions; - return $this->output; + if ($this->curlErrors !== []) { + [$this->lastCurlError, $message] = array_shift($this->curlErrors); + + throw HTTPException::forCurlError((string) $this->lastCurlError, $message); + } + + return $this->outputs !== [] ? array_shift($this->outputs) : $this->output; + } + + protected function sleep(float $seconds): void + { + $this->sleeps[] = $seconds; + } + + /** + * for testing purposes only + * + * @return list + */ + public function getSleeps(): array + { + return $this->sleeps; } /** diff --git a/tests/system/HTTP/CURLRequestRetryTest.php b/tests/system/HTTP/CURLRequestRetryTest.php new file mode 100644 index 000000000000..a8e36e3b7f3a --- /dev/null +++ b/tests/system/HTTP/CURLRequestRetryTest.php @@ -0,0 +1,344 @@ + + * + * For the full copyright and license information, please view + * the LICENSE file that was distributed with this source code. + */ + +namespace CodeIgniter\HTTP; + +use CodeIgniter\Config\Factories; +use CodeIgniter\Config\Services; +use CodeIgniter\HTTP\Exceptions\HTTPException; +use CodeIgniter\Superglobals; +use CodeIgniter\Test\CIUnitTestCase; +use CodeIgniter\Test\Mock\MockCURLRequest; +use Config\App; +use Config\CURLRequest as ConfigCURLRequest; +use PHPUnit\Framework\Attributes\Group; + +/** + * @internal + */ +#[Group('Others')] +final class CURLRequestRetryTest extends CIUnitTestCase +{ + private MockCURLRequest $request; + + protected function setUp(): void + { + parent::setUp(); + + $this->resetServices(); + Services::injectMock('superglobals', new Superglobals()); + $this->request = $this->getRequest(); + } + + /** + * @param array $options + */ + private function getRequest(array $options = []): MockCURLRequest + { + $uri = new URI($options['baseURI'] ?? null); + + $config = new ConfigCURLRequest(); + $config->shareOptions = false; + + Factories::injectMock('config', 'CURLRequest', $config); + + return new MockCURLRequest(new App(), $uri, new Response(), $options); + } + + public function testRetryIntegerRetriesDefaultStatusCodes(): void + { + $this->request->setOutputs([ + "HTTP/1.1 503 Service Unavailable\r\n\r\nFirst failure", + "HTTP/1.1 200 OK\r\n\r\nSuccess", + ]); + + $response = $this->request->get('http://example.com', [ + 'retry' => 3, + 'http_errors' => false, + ]); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame('Success', $response->getBody()); + $this->assertSame([1.0], $this->request->getSleeps()); + } + + public function testRetryIntegerRetriesDefaultGatewayTimeoutStatusCode(): void + { + $this->request->setOutputs([ + "HTTP/1.1 504 Gateway Timeout\r\n\r\nFirst failure", + "HTTP/1.1 200 OK\r\n\r\nSuccess", + ]); + + $response = $this->request->get('http://example.com', [ + 'retry' => 1, + 'http_errors' => false, + ]); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame('Success', $response->getBody()); + $this->assertSame([1.0], $this->request->getSleeps()); + } + + public function testRetryUsesCustomStatusCodes(): void + { + $this->request->setOutputs([ + "HTTP/1.1 500 Internal Server Error\r\n\r\nFirst failure", + "HTTP/1.1 200 OK\r\n\r\nSuccess", + ]); + + $response = $this->request->get('http://example.com', [ + 'retry' => [ + 'max_retries' => 1, + 'delay' => 100, + 'status_codes' => [500], + ], + 'http_errors' => false, + ]); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame([0.1], $this->request->getSleeps()); + } + + public function testRetryDoesNotRetryUnconfiguredStatusCode(): void + { + $this->request->setOutputs([ + "HTTP/1.1 404 Not Found\r\n\r\nMissing", + "HTTP/1.1 200 OK\r\n\r\nSuccess", + ]); + + $response = $this->request->get('http://example.com', [ + 'retry' => 3, + 'http_errors' => false, + ]); + + $this->assertSame(404, $response->getStatusCode()); + $this->assertSame('Missing', $response->getBody()); + $this->assertSame([], $this->request->getSleeps()); + } + + public function testZeroRetriesDisableRetryHandling(): void + { + $this->request->setOutput("HTTP/1.1 200 OK\r\n\r\nSuccess"); + + $this->request->get('http://example.com', [ + 'retry' => ['max_retries' => 0], + ]); + + $this->assertTrue($this->request->curl_options[CURLOPT_FAILONERROR]); + $this->assertSame([], $this->request->getSleeps()); + } + + public function testRetryUsesDelayBackoffArray(): void + { + $this->request->setOutputs([ + "HTTP/1.1 503 Service Unavailable\r\n\r\nFirst failure", + "HTTP/1.1 503 Service Unavailable\r\n\r\nSecond failure", + "HTTP/1.1 503 Service Unavailable\r\n\r\nThird failure", + "HTTP/1.1 200 OK\r\n\r\nSuccess", + ]); + + $response = $this->request->get('http://example.com', [ + 'retry' => [ + 'max_retries' => 3, + 'delay' => [100, 500], + ], + 'http_errors' => false, + ]); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame([0.1, 0.5, 0.5], $this->request->getSleeps()); + } + + public function testRetryClampsNegativeDelays(): void + { + $this->request->setOutputs([ + "HTTP/1.1 503 Service Unavailable\r\n\r\nFirst failure", + "HTTP/1.1 200 OK\r\n\r\nSuccess", + ]); + + $response = $this->request->get('http://example.com', [ + 'retry' => [ + 'max_retries' => 1, + 'delay' => -100, + ], + 'http_errors' => false, + ]); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame([0.0], $this->request->getSleeps()); + } + + public function testRetryAfterSecondsOverridesConfiguredDelay(): void + { + $this->request->setOutputs([ + "HTTP/1.1 429 Too Many Requests\r\nRetry-After: 2\r\n\r\nRate limited", + "HTTP/1.1 200 OK\r\n\r\nSuccess", + ]); + + $response = $this->request->get('http://example.com', [ + 'retry' => [ + 'max_retries' => 1, + 'delay' => 100, + ], + 'http_errors' => false, + ]); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame([2.0], $this->request->getSleeps()); + } + + public function testRetryAfterDateOverridesConfiguredDelay(): void + { + $retryAfter = gmdate('D, d M Y H:i:s', time() + 3600) . ' GMT'; + + $this->request->setOutputs([ + "HTTP/1.1 503 Service Unavailable\r\nRetry-After: {$retryAfter}\r\n\r\nUnavailable", + "HTTP/1.1 200 OK\r\n\r\nSuccess", + ]); + + $response = $this->request->get('http://example.com', [ + 'retry' => [ + 'max_retries' => 1, + 'delay' => 100, + 'max_delay' => 5000, + ], + 'http_errors' => false, + ]); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame([5.0], $this->request->getSleeps()); + } + + public function testRetryAfterIsCappedByDefaultMaxDelay(): void + { + $this->request->setOutputs([ + "HTTP/1.1 429 Too Many Requests\r\nRetry-After: 3600\r\n\r\nRate limited", + "HTTP/1.1 200 OK\r\n\r\nSuccess", + ]); + + $response = $this->request->get('http://example.com', [ + 'retry' => 1, + 'http_errors' => false, + ]); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame([30.0], $this->request->getSleeps()); + } + + public function testRetryAfterCanBeDisabled(): void + { + $this->request->setOutputs([ + "HTTP/1.1 429 Too Many Requests\r\nRetry-After: 2\r\n\r\nRate limited", + "HTTP/1.1 200 OK\r\n\r\nSuccess", + ]); + + $response = $this->request->get('http://example.com', [ + 'retry' => [ + 'max_retries' => 1, + 'delay' => 100, + 'respect_retry_after' => false, + ], + 'http_errors' => false, + ]); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame([0.1], $this->request->getSleeps()); + } + + public function testRetryThrowsAfterExhaustingRetriesWhenHttpErrorsEnabled(): void + { + $this->request->setOutputs([ + "HTTP/1.1 503 Service Unavailable\r\n\r\nFirst failure", + "HTTP/1.1 503 Service Unavailable\r\n\r\nFinal failure", + ]); + + $this->expectException(HTTPException::class); + $this->expectExceptionMessage('22 : The requested URL returned error: 503'); + + $this->request->get('http://example.com', [ + 'retry' => [ + 'max_retries' => 1, + 'delay' => 100, + ], + ]); + } + + public function testRetryReturnsFinalResponseAfterExhaustingRetriesWhenHttpErrorsDisabled(): void + { + $this->request->setOutputs([ + "HTTP/1.1 503 Service Unavailable\r\n\r\nFirst failure", + "HTTP/1.1 503 Service Unavailable\r\n\r\nFinal failure", + ]); + + $response = $this->request->get('http://example.com', [ + 'retry' => [ + 'max_retries' => 1, + 'delay' => 100, + ], + 'http_errors' => false, + ]); + + $this->assertSame(503, $response->getStatusCode()); + $this->assertSame('Final failure', $response->getBody()); + $this->assertSame([0.1], $this->request->getSleeps()); + } + + public function testCurlErrorsAreNotRetriedByDefault(): void + { + $this->request->setCurlErrors([ + [CURLE_OPERATION_TIMEDOUT, 'Operation timed out'], + ]); + + $this->expectException(HTTPException::class); + + $this->request->get('http://example.com', [ + 'retry' => 3, + ]); + } + + public function testCurlErrorsCanBeRetried(): void + { + $this->request->setCurlErrors([ + [CURLE_OPERATION_TIMEDOUT, 'Operation timed out'], + ])->setOutput("HTTP/1.1 200 OK\r\n\r\nSuccess"); + + $response = $this->request->get('http://example.com', [ + 'retry' => [ + 'max_retries' => 1, + 'delay' => 100, + 'curl_errors' => true, + ], + ]); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame('Success', $response->getBody()); + $this->assertSame([0.1], $this->request->getSleeps()); + } + + public function testNonTransientCurlErrorsAreNotRetried(): void + { + $this->request->setCurlErrors([ + [CURLE_UNSUPPORTED_PROTOCOL, 'Unsupported protocol'], + ]); + + $this->expectException(HTTPException::class); + + $this->request->get('http://example.com', [ + 'retry' => [ + 'max_retries' => 1, + 'delay' => 100, + 'curl_errors' => true, + ], + ]); + } +} diff --git a/user_guide_src/source/changelogs/v4.8.0.rst b/user_guide_src/source/changelogs/v4.8.0.rst index ca51e62004e9..da7070f07ca8 100644 --- a/user_guide_src/source/changelogs/v4.8.0.rst +++ b/user_guide_src/source/changelogs/v4.8.0.rst @@ -239,6 +239,7 @@ Helpers and Functions HTTP ==== +- Added the ``retry`` option to ``CURLRequest`` for retrying failed responses with configurable delays, retryable status codes, optional transient cURL error retries, and ``Retry-After`` support. See :ref:`curlrequest-request-options-retry`. - Added :ref:`Form Requests ` - a new ``FormRequest`` base class that encapsulates validation rules, custom error messages, and authorization logic for a single HTTP request. - Added ``SSEResponse`` class for streaming Server-Sent Events (SSE) over HTTP. See :ref:`server-sent-events`. - ``Response`` and its child classes no longer require ``Config\App`` passed to their constructors. diff --git a/user_guide_src/source/libraries/curlrequest.rst b/user_guide_src/source/libraries/curlrequest.rst index c1bbc4eccdfd..61e73e702796 100644 --- a/user_guide_src/source/libraries/curlrequest.rst +++ b/user_guide_src/source/libraries/curlrequest.rst @@ -374,6 +374,60 @@ You can pass along data to send as query string variables by passing an associat .. literalinclude:: curlrequest/029.php +.. _curlrequest-request-options-retry: + +retry +===== + +.. versionadded:: 4.8.0 + +The ``retry`` option retries failed requests before returning the final response or throwing an exception. +For simple cases, set ``retry`` to the number of retry attempts: + +.. code-block:: php + + $response = $client->request('GET', 'https://api.example.com/items', [ + 'retry' => 3, + ]); + +For more control, pass an array: + +.. literalinclude:: curlrequest/041.php + +The available retry settings are: + +- ``max_retries``: Number of retries after the initial request. The default is ``3``. +- ``delay``: Delay in milliseconds before retrying. This may be an integer or a list of integers + for simple backoff. If the list is shorter than the retry count, the last value is reused. + The default is ``1000``. +- ``max_delay``: Maximum delay in milliseconds. This caps both the configured ``delay`` and a + valid ``Retry-After`` header. Set to ``0`` for no maximum delay. The default is ``30000``. +- ``status_codes``: HTTP status codes that should be retried. The default is ``[429, 503, 504]``. +- ``curl_errors``: Whether to retry transient cURL errors. The default is ``false``. +- ``respect_retry_after``: Whether to use a valid ``Retry-After`` header instead of the configured + ``delay``. The default is ``true``. + +When ``respect_retry_after`` is enabled, a valid ``Retry-After`` header takes priority over the +configured ``delay``. The header may be either a number of seconds or an HTTP date. If the header +is invalid, the configured ``delay`` is used instead. + +Because ``Retry-After`` is supplied by the remote server, it is capped by ``max_delay`` by default +to avoid unexpectedly long waits in the current PHP process. + +When ``curl_errors`` is enabled, only DNS resolution failures, connection failures, timeouts, +and send or receive failures are retried. + +When `http_errors`_ is enabled, an HTTP error response is retried first if its status code is +configured in ``status_codes``. If all retry attempts are exhausted, the final HTTP error response +throws the same as a request without retries. + +.. note:: Retry delays block the current PHP process. The total request time may exceed the + configured `timeout`_ because each retry attempt has its own cURL timeout and retry delays + are added between attempts. + +.. warning:: Be careful when retrying non-idempotent requests such as ``POST`` or ``PATCH``. + The remote server may receive the request more than once. + timeout ======= diff --git a/user_guide_src/source/libraries/curlrequest/041.php b/user_guide_src/source/libraries/curlrequest/041.php new file mode 100644 index 000000000000..2044c4f31f98 --- /dev/null +++ b/user_guide_src/source/libraries/curlrequest/041.php @@ -0,0 +1,12 @@ +request('GET', 'https://api.example.com/items', [ + 'retry' => [ + 'max_retries' => 3, + 'delay' => [100, 500, 1000], + 'max_delay' => 5000, + 'status_codes' => [429, 500, 502, 503, 504], + 'curl_errors' => true, + 'respect_retry_after' => true, + ], +]);