From ce20ca12ab2a931c78c838068dce654e2a8c3d40 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Wed, 15 Oct 2025 11:22:37 +1100 Subject: [PATCH] Rate limit write load decider logging --- .../WriteLoadConstraintSettings.java | 10 +++++++++ .../decider/WriteLoadConstraintDecider.java | 22 ++++++++++++++----- .../common/settings/ClusterSettings.java | 3 ++- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java index 156f1d30d66c0..62984715e9d15 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java @@ -109,6 +109,16 @@ public boolean disabled() { Setting.Property.NodeScope ); + /** + * The minimum amount of time between logging messages about write load decider interventions + */ + public static final Setting WRITE_LOAD_DECIDER_MINIMUM_LOGGING_INTERVAL = Setting.timeSetting( + SETTING_PREFIX + "log_interval", + TimeValue.timeValueMinutes(1), + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + private volatile WriteLoadDeciderStatus writeLoadDeciderStatus; private volatile TimeValue minimumRerouteInterval; private volatile double highUtilizationThreshold; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDecider.java index ec7471f3ca147..08c661adf37e7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDecider.java @@ -18,8 +18,10 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.WriteLoadConstraintSettings; +import org.elasticsearch.common.FrequencyCappedAction; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.core.Strings; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.threadpool.ThreadPool; /** @@ -31,10 +33,16 @@ public class WriteLoadConstraintDecider extends AllocationDecider { public static final String NAME = "write_load"; + private final FrequencyCappedAction logInterventionMessage; private final WriteLoadConstraintSettings writeLoadConstraintSettings; public WriteLoadConstraintDecider(ClusterSettings clusterSettings) { this.writeLoadConstraintSettings = new WriteLoadConstraintSettings(clusterSettings); + logInterventionMessage = new FrequencyCappedAction(System::currentTimeMillis, TimeValue.ZERO); + clusterSettings.initializeAndWatch( + WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_MINIMUM_LOGGING_INTERVAL, + logInterventionMessage::setMinInterval + ); } @Override @@ -77,7 +85,9 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing nodeWriteThreadPoolLoadThreshold, shardRouting.shardId() ); - logger.debug(explain); + if (logger.isDebugEnabled()) { + logInterventionMessage.maybeExecute(() -> logger.debug(explain)); + } return Decision.single(Decision.Type.NOT_PREFERRED, NAME, explain); } @@ -97,7 +107,9 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing shardWriteLoad, nodeWriteThreadPoolStats.totalThreadPoolThreads() ); - logger.debug(explain); + if (logger.isDebugEnabled()) { + logInterventionMessage.maybeExecute(() -> logger.debug(explain)); + } return Decision.single(Decision.Type.NOT_PREFERRED, NAME, explain); } @@ -108,7 +120,6 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing node.nodeId(), newWriteThreadPoolUtilization ); - logger.trace(explanation); return allocation.decision(Decision.YES, NAME, explanation); } @@ -146,7 +157,9 @@ public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting nodeWriteThreadPoolQueueLatencyThreshold.toHumanReadableString(2), nodeWriteThreadPoolStats.averageThreadPoolUtilization() ); - logger.debug(explain); + if (logger.isDebugEnabled()) { + logInterventionMessage.maybeExecute(() -> logger.debug(explain)); + } return Decision.single(Decision.Type.NOT_PREFERRED, NAME, explain); } @@ -156,7 +169,6 @@ public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting nodeWriteThreadPoolStats.maxThreadPoolQueueLatencyMillis(), nodeWriteThreadPoolQueueLatencyThreshold.toHumanReadableString(2) ); - logger.trace(explanation); return allocation.decision(Decision.YES, NAME, explanation); } diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 7006b5adbe886..cd87464a3931c 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -650,6 +650,7 @@ public void apply(Settings value, Settings current, Settings previous) { WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_HIGH_UTILIZATION_THRESHOLD_SETTING, WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_HIGH_UTILIZATION_DURATION_SETTING, WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_QUEUE_LATENCY_THRESHOLD_SETTING, - WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_REROUTE_INTERVAL_SETTING + WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_REROUTE_INTERVAL_SETTING, + WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_MINIMUM_LOGGING_INTERVAL ); }