diff --git a/go.mod b/go.mod index a538132c..a9893723 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.26 require ( github.com/KimMachineGun/automemlimit v0.7.5 - github.com/aws/aws-sdk-go-v2 v1.41.12 + github.com/aws/aws-sdk-go-v2 v1.42.0 github.com/aws/aws-sdk-go-v2/config v1.32.23 github.com/aws/aws-sdk-go-v2/credentials v1.19.22 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.25 diff --git a/go.sum b/go.sum index 3c9ac26c..54156a1a 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7X github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= -github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= +github.com/aws/aws-sdk-go-v2 v1.42.0 h1:XvXMJTkFQtpBKIWZnmr9ZEOc2InWM2yldjXEJ/bymhA= +github.com/aws/aws-sdk-go-v2 v1.42.0/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 h1:p1BBrg/Hhp6uK7zpejeI8QFXHJeC/mynzi04Sl03k9g= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13/go.mod h1:8cIfkE9MDhkRZGpQ22aV6/lkYeYSozpz16Smrs5x4Ls= github.com/aws/aws-sdk-go-v2/config v1.32.23 h1:PYDobtcsJXK6bQe9I8RQk6s19Bz3xa3xRU08Hy1Em3Y= diff --git a/vendor/github.com/aws/aws-sdk-go-v2/aws/go_module_metadata.go b/vendor/github.com/aws/aws-sdk-go-v2/aws/go_module_metadata.go index ff84834d..46cb77c2 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/aws/go_module_metadata.go +++ b/vendor/github.com/aws/aws-sdk-go-v2/aws/go_module_metadata.go @@ -3,4 +3,4 @@ package aws // goModuleVersion is the tagged release for this module -const goModuleVersion = "1.41.12" +const goModuleVersion = "1.42.0" diff --git a/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/jitter_backoff.go b/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/jitter_backoff.go index c266996d..14225a53 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/jitter_backoff.go +++ b/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/jitter_backoff.go @@ -4,6 +4,7 @@ import ( "math" "time" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/internal/rand" "github.com/aws/aws-sdk-go-v2/internal/timeconv" ) @@ -12,9 +13,20 @@ import ( // number of attempts. type ExponentialJitterBackoff struct { maxBackoff time.Duration - // precomputed number of attempts needed to reach max backoff. + // precomputed number of attempts needed to reach max backoff (legacy mode). maxBackoffAttempts float64 + // Base delay for non-throttle errors (x in the formula t_i = b * min(x * r^i, MAX_BACKOFF)). + baseDelay time.Duration + + // Throttle error checker. When set and the error is a throttle, the base + // delay is 1s regardless of the configured baseDelay. + throttle IsErrorThrottle + + // When true, applies MAX_BACKOFF before jitter and uses throttle-aware + // base delay. + retries2026 bool + randFloat64 func() (float64, error) } @@ -25,13 +37,53 @@ func NewExponentialJitterBackoff(maxBackoff time.Duration) *ExponentialJitterBac maxBackoff: maxBackoff, maxBackoffAttempts: math.Log2( float64(maxBackoff) / float64(time.Second)), + baseDelay: time.Second, randFloat64: rand.CryptoRandFloat64, } } +// exponentialJitterBackoffOption is a functional option for ExponentialJitterBackoff. +type exponentialJitterBackoffOption func(*ExponentialJitterBackoff) + +// withBaseDelay sets the base delay for non-throttle errors. +func withBaseDelay(d time.Duration) exponentialJitterBackoffOption { + return func(j *ExponentialJitterBackoff) { + j.baseDelay = d + } +} + +// withThrottleCheck sets the throttle error checker used to determine if the +// backoff should use the throttle base delay (1s) instead of the configured +// base delay. +func withThrottleCheck(t IsErrorThrottle) exponentialJitterBackoffOption { + return func(j *ExponentialJitterBackoff) { + j.throttle = t + } +} + +// newExponentialJitterBackoffWithOptions returns an ExponentialJitterBackoff +// with the given options applied. +func newExponentialJitterBackoffWithOptions(maxBackoff time.Duration, optFns ...exponentialJitterBackoffOption) *ExponentialJitterBackoff { + j := NewExponentialJitterBackoff(maxBackoff) + j.retries2026 = true + for _, fn := range optFns { + fn(j) + } + return j +} + // BackoffDelay returns the duration to wait before the next attempt should be // made. Returns an error if unable get a duration. func (j *ExponentialJitterBackoff) BackoffDelay(attempt int, err error) (time.Duration, error) { + if j.retries2026 { + return j.backoffDelay2026(attempt, err) + } + return j.backoffDelayLegacy(attempt, err) +} + +// backoffDelayLegacy preserves the original backoff formula: b * 2^i, capped +// at maxBackoff. +func (j *ExponentialJitterBackoff) backoffDelayLegacy(attempt int, err error) (time.Duration, error) { if attempt > int(j.maxBackoffAttempts) { return j.maxBackoff, nil } @@ -47,3 +99,26 @@ func (j *ExponentialJitterBackoff) BackoffDelay(attempt int, err error) (time.Du return timeconv.FloatSecondsDur(delaySeconds), nil } + +// backoffDelay2026 uses throttle-aware base delay and applies MAX_BACKOFF +// before jitter: t_i = b * min(x * 2^i, MAX_BACKOFF). +func (j *ExponentialJitterBackoff) backoffDelay2026(attempt int, err error) (time.Duration, error) { + x := j.baseDelay + if j.throttle != nil && j.throttle.IsErrorThrottle(err) == aws.TrueTernary { + x = time.Second + } + + b, randErr := j.randFloat64() + if randErr != nil { + return 0, randErr + } + + ri := math.Pow(2, float64(attempt)) + delaySeconds := float64(x) / float64(time.Second) * ri + maxBackoffSeconds := float64(j.maxBackoff) / float64(time.Second) + if delaySeconds > maxBackoffSeconds { + delaySeconds = maxBackoffSeconds + } + + return timeconv.FloatSecondsDur(b * delaySeconds), nil +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/middleware.go b/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/middleware.go index 52acb62f..ab024de0 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/middleware.go +++ b/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/middleware.go @@ -233,9 +233,11 @@ func (r *Attempt) handleAttempt( "failed to release retry token after request error, %w", err) } // Release the attempt token based on the state of the attempt's error (if any). - if releaseError := releaseAttemptToken(err); releaseError != nil && err != nil { - return out, attemptResult, nopRelease, fmt.Errorf( - "failed to release initial token after request error, %w", err) + if !newRetries2026() || attemptNum == 1 { + if releaseError := releaseAttemptToken(err); releaseError != nil && err != nil { + return out, attemptResult, nopRelease, fmt.Errorf( + "failed to release initial token after request error, %w", err) + } } // If there was no error making the attempt, nothing further to do. There // will be nothing to retry. @@ -276,6 +278,13 @@ func (r *Attempt) handleAttempt( // Get a retry token that will be released after the releaseRetryToken, retryTokenErr := r.retryer.GetRetryToken(ctx, err) if retryTokenErr != nil { + // Long-polling operations must still back off when quota is exceeded. + if newRetries2026() && internalcontext.GetIsLongPolling(ctx) { + if retryDelay, delayErr := r.retryer.RetryDelay(attemptNum-1, err); delayErr == nil { + retryDelay = adjustForRetryAfterHeader(retryDelay, err, logger, r.LogAttempts) + _ = sdk.SleepWithContext(ctx, retryDelay) + } + } return out, attemptResult, nopRelease, errors.Join(err, retryTokenErr) } @@ -285,10 +294,17 @@ func (r *Attempt) handleAttempt( // Get the retry delay before another attempt can be made, and sleep for // that time. Potentially early exist if the sleep is canceled via the // context. - retryDelay, reqErr := r.retryer.RetryDelay(attemptNum, err) + attempt := attemptNum + if newRetries2026() { + attempt = attemptNum - 1 + } + retryDelay, reqErr := r.retryer.RetryDelay(attempt, err) if reqErr != nil { return out, attemptResult, releaseRetryToken, reqErr } + if newRetries2026() { + retryDelay = adjustForRetryAfterHeader(retryDelay, err, logger, r.LogAttempts) + } if reqErr = sdk.SleepWithContext(ctx, retryDelay); reqErr != nil { err = &aws.RequestCanceledError{Err: reqErr} return out, attemptResult, releaseRetryToken, err @@ -423,6 +439,43 @@ func AddRetryMiddlewares(stack *smithymiddle.Stack, options AddRetryMiddlewaresO return nil } +// adjustForRetryAfterHeader checks for the x-amz-retry-after response header +// and clamps the backoff duration accordingly. The header value is an integer +// representing milliseconds. The result is clamped to [t_i, 5s + t_i] where +// t_i is the jittered exponential backoff duration. Invalid header values are +// ignored. +func adjustForRetryAfterHeader(backoff time.Duration, err error, logger logging.Logger, logAttempts bool) time.Duration { + var re *http.ResponseError + if !errors.As(err, &re) || re.Response == nil || re.Response.Response == nil { + return backoff + } + + headerVal := re.Response.Header.Get("X-Amz-Retry-After") + if headerVal == "" { + return backoff + } + + ms, parseErr := strconv.ParseInt(headerVal, 10, 64) + if parseErr != nil || ms < 0 { + if logAttempts { + logger.Logf(logging.Debug, "ignoring invalid x-amz-retry-after header value %q", headerVal) + } + return backoff + } + + retryAfter := time.Duration(ms) * time.Millisecond + minDuration := backoff + maxDuration := 5*time.Second + backoff + + if retryAfter < minDuration { + return minDuration + } + if retryAfter > maxDuration { + return maxDuration + } + return retryAfter +} + // Determines the value of exception.type for metrics purposes. We prefer an // API-specific error code, otherwise it's just the Go type for the value. func errorType(err error) string { diff --git a/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/retry.go b/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/retry.go index af81635b..c240fb09 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/retry.go +++ b/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/retry.go @@ -72,6 +72,19 @@ func (r *withMaxBackoffDelay) RetryDelay(attempt int, err error) (time.Duration, return r.backoff.BackoffDelay(attempt, err) } +// AddWithLongPolling returns a retryer that is marked as long-polling. +// Long-polling operations will back off even when the retry quota is +// exhausted. +func AddWithLongPolling(r aws.Retryer) aws.Retryer { + return &withLongPolling{RetryerV2: wrapAsRetryerV2(r)} +} + +type withLongPolling struct { + aws.RetryerV2 +} + +func (w *withLongPolling) IsLongPolling() bool { return true } + type wrappedAsRetryerV2 struct { aws.Retryer } diff --git a/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/standard.go b/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/standard.go index d5ea9322..f2f9660d 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/standard.go +++ b/vendor/github.com/aws/aws-sdk-go-v2/aws/retry/standard.go @@ -3,6 +3,7 @@ package retry import ( "context" "fmt" + "os" "time" "github.com/aws/aws-sdk-go-v2/aws/ratelimit" @@ -35,8 +36,16 @@ const ( const ( DefaultRetryRateTokens uint = 500 DefaultRetryCost uint = 5 - DefaultRetryTimeoutCost uint = 10 DefaultNoRetryIncrement uint = 1 + + // DefaultRetryTimeoutCost is the cost to deduct from the RateLimiter's + // token bucket per retry caused by timeout error. + // + // When AWS_NEW_RETRIES_2026 is set to "true", timeouts are no longer + // treated differently than other transient errors. The discounted cost + // is instead applied to throttling errors via DefaultThrottlingRetryCost. + DefaultRetryTimeoutCost uint = 10 + DefaultThrottlingRetryCost uint = 5 ) // DefaultRetryableHTTPStatusCodes is the default set of HTTP status codes the SDK @@ -121,6 +130,12 @@ type StandardOptions struct { // It is safe to append to this list in NewStandard's functional options. Timeouts []IsErrorTimeout + // Set of strategies to determine if the attempt failed due to a throttle + // error. Used to determine the retry token cost. + // + // It is safe to append to this list in NewStandard's functional options. + Throttles []IsErrorThrottle + // Provides the rate limiting strategy for rate limiting attempt retries // across all attempts the retryer is being used with. // @@ -129,10 +144,14 @@ type StandardOptions struct { // consume more tokens than what's available results in operation failure. // The default implementation is parameterized as follows: // - a capacity of 500 (DefaultRetryRateTokens) - // - a retry caused by a timeout costs 10 tokens (DefaultRetryCost) - // - a retry caused by other errors costs 5 tokens (DefaultRetryTimeoutCost) + // - a retry caused by a timeout costs 10 tokens (DefaultRetryTimeoutCost) + // - a retry caused by other errors costs 5 tokens (DefaultRetryCost) // - an operation that succeeds on the 1st attempt adds 1 token (DefaultNoRetryIncrement) // + // When AWS_NEW_RETRIES_2026 is set to "true", the costs change: + // - a retry costs 14 tokens + // - a retry caused by a throttling error costs 5 tokens (DefaultThrottlingRetryCost) + // // You can disable rate limiting by setting this field to ratelimit.None. RateLimiter RateLimiter @@ -141,11 +160,23 @@ type StandardOptions struct { // The cost to deduct from the RateLimiter's token bucket per retry caused // by timeout error. + // + // When AWS_NEW_RETRIES_2026 is set to "true", this field is unused. + // Throttling errors use ThrottlingRetryCost instead. RetryTimeoutCost uint + // The cost to deduct from the RateLimiter's token bucket per retry caused + // by a throttling error. Only used when AWS_NEW_RETRIES_2026 is "true". + ThrottlingRetryCost uint + // The cost to payback to the RateLimiter's token bucket for successful // attempts. NoRetryIncrement uint + + // BaseDelay is the base backoff delay for non-throttle retryable errors. + // Throttling errors always use 1s. Defaults to 50ms if zero. + // Only used when AWS_NEW_RETRIES_2026 is "true"; ignored in legacy mode. + BaseDelay time.Duration } // RateLimiter provides the interface for limiting the rate of attempt retries @@ -161,6 +192,7 @@ type RateLimiter interface { type Standard struct { options StandardOptions + throttle IsErrorThrottle timeout IsErrorTimeout retryable IsErrorRetryable backoff BackoffDelayer @@ -169,17 +201,7 @@ type Standard struct { // NewStandard initializes a standard retry behavior with defaults that can be // overridden via functional options. func NewStandard(fnOpts ...func(*StandardOptions)) *Standard { - o := StandardOptions{ - MaxAttempts: DefaultMaxAttempts, - MaxBackoff: DefaultMaxBackoff, - Retryables: append([]IsErrorRetryable{}, DefaultRetryables...), - Timeouts: append([]IsErrorTimeout{}, DefaultTimeouts...), - - RateLimiter: ratelimit.NewTokenRateLimit(DefaultRetryRateTokens), - RetryCost: DefaultRetryCost, - RetryTimeoutCost: DefaultRetryTimeoutCost, - NoRetryIncrement: DefaultNoRetryIncrement, - } + o := standardDefaults() for _, fn := range fnOpts { fn(&o) } @@ -189,13 +211,25 @@ func NewStandard(fnOpts ...func(*StandardOptions)) *Standard { backoff := o.Backoff if backoff == nil { - backoff = NewExponentialJitterBackoff(o.MaxBackoff) + if newRetries2026() { + baseDelay := o.BaseDelay + if baseDelay == 0 { + baseDelay = 50 * time.Millisecond + } + backoff = newExponentialJitterBackoffWithOptions(o.MaxBackoff, + withBaseDelay(baseDelay), + withThrottleCheck(IsErrorThrottles(o.Throttles)), + ) + } else { + backoff = NewExponentialJitterBackoff(o.MaxBackoff) + } } return &Standard{ options: o, backoff: backoff, retryable: IsErrorRetryables(o.Retryables), + throttle: IsErrorThrottles(o.Throttles), timeout: IsErrorTimeouts(o.Timeouts), } } @@ -244,8 +278,14 @@ func (s *Standard) noRetryIncrement() error { func (s *Standard) GetRetryToken(ctx context.Context, opErr error) (func(error) error, error) { cost := s.options.RetryCost - if s.timeout.IsErrorTimeout(opErr).Bool() { - cost = s.options.RetryTimeoutCost + if newRetries2026() { + if s.throttle.IsErrorThrottle(opErr).Bool() { + cost = s.options.ThrottlingRetryCost + } + } else { + if s.timeout.IsErrorTimeout(opErr).Bool() { + cost = s.options.RetryTimeoutCost + } } fn, err := s.options.RateLimiter.GetToken(ctx, cost) @@ -267,3 +307,37 @@ func (f releaseToken) release(err error) error { return f() } + +func newRetries2026() bool { + return os.Getenv("AWS_NEW_RETRIES_2026") == "true" +} + +func standardDefaults() StandardOptions { + if newRetries2026() { + return StandardOptions{ + MaxAttempts: DefaultMaxAttempts, + MaxBackoff: DefaultMaxBackoff, + Retryables: append([]IsErrorRetryable{}, DefaultRetryables...), + Timeouts: append([]IsErrorTimeout{}, DefaultTimeouts...), + Throttles: append([]IsErrorThrottle{}, DefaultThrottles...), + + RateLimiter: ratelimit.NewTokenRateLimit(DefaultRetryRateTokens), + RetryCost: 14, + RetryTimeoutCost: DefaultRetryTimeoutCost, + ThrottlingRetryCost: DefaultThrottlingRetryCost, + NoRetryIncrement: DefaultNoRetryIncrement, + } + } + return StandardOptions{ + MaxAttempts: DefaultMaxAttempts, + MaxBackoff: DefaultMaxBackoff, + Retryables: append([]IsErrorRetryable{}, DefaultRetryables...), + Timeouts: append([]IsErrorTimeout{}, DefaultTimeouts...), + Throttles: append([]IsErrorThrottle{}, DefaultThrottles...), + + RateLimiter: ratelimit.NewTokenRateLimit(DefaultRetryRateTokens), + RetryCost: DefaultRetryCost, + RetryTimeoutCost: DefaultRetryTimeoutCost, + NoRetryIncrement: DefaultNoRetryIncrement, + } +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/internal/context/context.go b/vendor/github.com/aws/aws-sdk-go-v2/internal/context/context.go index f0c283d3..52f4ebc2 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/internal/context/context.go +++ b/vendor/github.com/aws/aws-sdk-go-v2/internal/context/context.go @@ -50,3 +50,16 @@ func GetAttemptSkewContext(ctx context.Context) time.Duration { x, _ := middleware.GetStackValue(ctx, clockSkew{}).(time.Duration) return x } + +type longPollingKey struct{} + +// SetIsLongPolling marks the operation as long-polling on the context. +func SetIsLongPolling(ctx context.Context, v bool) context.Context { + return middleware.WithStackValue(ctx, longPollingKey{}, v) +} + +// GetIsLongPolling returns whether the operation is long-polling. +func GetIsLongPolling(ctx context.Context) bool { + v, _ := middleware.GetStackValue(ctx, longPollingKey{}).(bool) + return v +} diff --git a/vendor/modules.txt b/vendor/modules.txt index d8389211..2594249a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -15,7 +15,7 @@ github.com/STARRY-S/zip ## explicit; go 1.13 github.com/andybalholm/brotli github.com/andybalholm/brotli/matchfinder -# github.com/aws/aws-sdk-go-v2 v1.41.12 +# github.com/aws/aws-sdk-go-v2 v1.42.0 ## explicit; go 1.24 github.com/aws/aws-sdk-go-v2/aws github.com/aws/aws-sdk-go-v2/aws/arn