diff --git a/va/va.go b/va/va.go
index fd766262267..e2895c28023 100644
--- a/va/va.go
+++ b/va/va.go
@@ -571,6 +571,7 @@ func (va *ValidationAuthorityImpl) doRemoteOperation(ctx context.Context, op rem
 	var failed []string
 	var passedRIRs = map[string]struct{}{}
 	var firstProb *probs.ProblemDetails
+	var slowTimerSet bool
 
 	for resp := range responses {
 		var currProb *probs.ProblemDetails
@@ -606,13 +607,14 @@ func (va *ValidationAuthorityImpl) doRemoteOperation(ctx context.Context, op rem
 			firstProb = currProb
 		}
 
-		if va.slowRemoteTimeout != 0 {
+		if va.slowRemoteTimeout != 0 && !slowTimerSet {
 			// If enough perspectives have passed, or enough perspectives have
 			// failed, set a tighter deadline for the remaining perspectives.
 			if (len(passed) >= required && len(passedRIRs) >= requiredRIRs) ||
 				(len(failed) > remoteVACount-required) {
 				timer := time.AfterFunc(va.slowRemoteTimeout, cancel)
 				defer timer.Stop()
+				slowTimerSet = true
 			}
 		}