From 86b65a99fa1453d43c01e1dfa7e760c3c6f281ad Mon Sep 17 00:00:00 2001 From: Hazel Date: Thu, 10 Oct 2024 20:31:33 -0400 Subject: [PATCH 1/6] Add one IT for binary vector support --- .../api/v1/metrics/JsonApiMetricsConfig.java | 4 +++ .../processor/MeteredCommandProcessor.java | 25 ++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java index 96ca29bb89..6f908910ea 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java @@ -86,6 +86,10 @@ public interface JsonApiMetricsConfig { @WithDefault("index.usage.count") String indexUsageCounterMetrics(); + @NotBlank + @WithDefault("command.processor.latency") + String commandProcessorLatencyMetrics(); + /** List of values that can be used as value for metrics sort_type. */ enum SortType { // Uses vertor search sorting for document resolution diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java index 5cb882f597..875c8928b3 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java @@ -29,6 +29,7 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -110,9 +111,17 @@ public Uni processCom .onItem() .invoke( result -> { - Tags tags = getCustomTags(commandContext, command, result); - // add metrics - sample.stop(meterRegistry.timer(jsonApiMetricsConfig.metricsName(), tags)); + Tags complexTags = getCustomTags(commandContext, command, result); + Tags simpleTags = getSimpleTags(command); + // add command metrics with complex tags + long durationNs = + sample.stop(meterRegistry.timer(jsonApiMetricsConfig.metricsName(), complexTags)); + // add command metrics with simple tags for histogram (reassigned the timer to ensure + // two metrics are identical) + Timer.builder(jsonApiMetricsConfig.commandProcessorLatencyMetrics()) + .tags(simpleTags) + .register(meterRegistry) + .record(durationNs, TimeUnit.NANOSECONDS); if (isCommandLevelLoggingEnabled(result, false)) { logger.info(buildCommandLog(commandContext, command, result)); @@ -262,6 +271,13 @@ private Tags getCustomTags( return tags; } + private Tags getSimpleTags(Command command) { + Tag commandTag = Tag.of(jsonApiMetricsConfig.command(), command.getClass().getSimpleName()); + String tenant = dataApiRequestInfo.getTenantId().orElse(UNKNOWN_VALUE); + Tag tenantTag = Tag.of(tenantConfig.tenantTag(), tenant); + return Tags.of(commandTag, tenantTag); + } + private JsonApiMetricsConfig.SortType getVectorTypeTag(Command command) { int filterCount = 0; if (command instanceof Filterable fc && fc.filterClause() != null) { @@ -300,7 +316,8 @@ public MeterFilter enableHistogram() { public DistributionStatisticConfig configure( Meter.Id id, DistributionStatisticConfig config) { if (id.getName().startsWith(HISTOGRAM_METRICS_NAME) - || id.getName().startsWith(jsonApiMetricsConfig.vectorizeCallDurationMetrics())) { + || id.getName().startsWith(jsonApiMetricsConfig.vectorizeCallDurationMetrics()) + || id.getName().startsWith(jsonApiMetricsConfig.commandProcessorLatencyMetrics())) { return DistributionStatisticConfig.builder() .percentiles(0.5, 0.90, 0.95, 0.99) // median and 95th percentile, not aggregable From 36b3c84d5488774ee97f527b07a30564a43d4d17 Mon Sep 17 00:00:00 2001 From: Hazel Date: Thu, 10 Oct 2024 20:51:50 -0400 Subject: [PATCH 2/6] Add test --- .../MeteredCommandProcessorTest.java | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessorTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessorTest.java index 94166230a6..37a2b94699 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessorTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessorTest.java @@ -88,6 +88,43 @@ public void metrics() throws Exception { assertThat(line).contains("module=\"sgv2-jsonapi\""); }); }); + + List commandLatencyMetrics = + metrics + .lines() + .filter( + line -> + line.startsWith("command_processor_latency") + && line.contains("CountDocumentsCommand") + && !line.startsWith("command_processor_latency_seconds_bucket") + && !line.contains("quantile")) + .toList(); + assertThat(commandLatencyMetrics) + .satisfies( + lines -> { + assertThat(lines.size()).isEqualTo(3); + lines.forEach( + line -> { + assertThat(line).contains("command=\"CountDocumentsCommand\""); + assertThat(line).contains("tenant=\"test-tenant\""); + assertThat(line).contains("module=\"sgv2-jsonapi\""); + assertThat(line).doesNotContain("sort_type"); + assertThat(line).doesNotContain("error"); + assertThat(line).doesNotContain("error_code"); + assertThat(line).doesNotContain("error_class"); + assertThat(line).doesNotContain("vector_enabled"); + }); + }); + List commandLatencyHistogram = + metrics + .lines() + .filter(line -> line.startsWith("command_processor_latency_seconds_bucket")) + .toList(); + assertThat(commandLatencyHistogram) + .satisfies( + lines -> { + assertThat(lines.size()).isNotZero(); + }); } @Test From ce13c3cac2b26347ed2416f9ca6f6a4cf178537b Mon Sep 17 00:00:00 2001 From: Hazel Date: Mon, 14 Oct 2024 19:32:53 -0400 Subject: [PATCH 3/6] reduce the number of bucket --- .../processor/MeteredCommandProcessor.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java index 875c8928b3..eaac32f6e8 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java @@ -316,8 +316,7 @@ public MeterFilter enableHistogram() { public DistributionStatisticConfig configure( Meter.Id id, DistributionStatisticConfig config) { if (id.getName().startsWith(HISTOGRAM_METRICS_NAME) - || id.getName().startsWith(jsonApiMetricsConfig.vectorizeCallDurationMetrics()) - || id.getName().startsWith(jsonApiMetricsConfig.commandProcessorLatencyMetrics())) { + || id.getName().startsWith(jsonApiMetricsConfig.vectorizeCallDurationMetrics())) { return DistributionStatisticConfig.builder() .percentiles(0.5, 0.90, 0.95, 0.99) // median and 95th percentile, not aggregable @@ -325,6 +324,18 @@ public DistributionStatisticConfig configure( .build() .merge(config); } + + // reduce the number of buckets by setting the min and max expected values to avoid the high + // cardinality problem in Grafana + if (id.getName().startsWith(jsonApiMetricsConfig.commandProcessorLatencyMetrics())) { + return DistributionStatisticConfig.builder() + .percentiles(0.5, 0.90, 0.95, 0.99) + .percentilesHistogram(true) + .minimumExpectedValue(TimeUnit.MILLISECONDS.toNanos(100)) // 0.1 seconds + .maximumExpectedValue(TimeUnit.SECONDS.toNanos(15)) // 15 seconds + .build() + .merge(config); + } return config; } }; From 5b7b15819b837506e3f3cdbca57666c23f1a0727 Mon Sep 17 00:00:00 2001 From: Hazel Date: Mon, 14 Oct 2024 19:51:00 -0400 Subject: [PATCH 4/6] remove the hard coded value --- .../api/v1/metrics/JsonApiMetricsConfig.java | 16 ++++++++++++++++ .../processor/MeteredCommandProcessor.java | 8 ++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java index 6f908910ea..b51c9be2b9 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java @@ -90,6 +90,22 @@ public interface JsonApiMetricsConfig { @WithDefault("command.processor.latency") String commandProcessorLatencyMetrics(); + /** + * The minimum value that the meter is expected to observe for the command processor latency. The + * unit is milliseconds. + */ + @NotBlank + @WithDefault("100") + int MinExpectedCommandProcessorLatency(); + + /** + * The maximum value that the meter is expected to observe for the command processor latency. The + * unit is milliseconds. + */ + @NotBlank + @WithDefault("15000") + int MaxExpectedCommandProcessorLatency(); + /** List of values that can be used as value for metrics sort_type. */ enum SortType { // Uses vertor search sorting for document resolution diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java index eaac32f6e8..74fe8036bb 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java @@ -331,8 +331,12 @@ public DistributionStatisticConfig configure( return DistributionStatisticConfig.builder() .percentiles(0.5, 0.90, 0.95, 0.99) .percentilesHistogram(true) - .minimumExpectedValue(TimeUnit.MILLISECONDS.toNanos(100)) // 0.1 seconds - .maximumExpectedValue(TimeUnit.SECONDS.toNanos(15)) // 15 seconds + .minimumExpectedValue( + TimeUnit.MILLISECONDS.toNanos( + jsonApiMetricsConfig.MinExpectedCommandProcessorLatency())) // 0.1 seconds + .maximumExpectedValue( + TimeUnit.MILLISECONDS.toNanos( + jsonApiMetricsConfig.MaxExpectedCommandProcessorLatency())) // 15 seconds .build() .merge(config); } From 65ce427ee5f69266bc5a50ae034f1984eec19b75 Mon Sep 17 00:00:00 2001 From: Hazel Date: Mon, 14 Oct 2024 20:04:13 -0400 Subject: [PATCH 5/6] fix --- .../sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java index b51c9be2b9..2864d51af9 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java @@ -94,7 +94,6 @@ public interface JsonApiMetricsConfig { * The minimum value that the meter is expected to observe for the command processor latency. The * unit is milliseconds. */ - @NotBlank @WithDefault("100") int MinExpectedCommandProcessorLatency(); @@ -102,7 +101,6 @@ public interface JsonApiMetricsConfig { * The maximum value that the meter is expected to observe for the command processor latency. The * unit is milliseconds. */ - @NotBlank @WithDefault("15000") int MaxExpectedCommandProcessorLatency(); From 7244c62455e3ff2d421cc26b7bbeef1987484389 Mon Sep 17 00:00:00 2001 From: Hazel Date: Tue, 15 Oct 2024 12:08:40 -0400 Subject: [PATCH 6/6] Update the min --- .../sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java | 2 +- .../sgv2/jsonapi/service/processor/MeteredCommandProcessor.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java index 2864d51af9..903851a9b0 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java @@ -94,7 +94,7 @@ public interface JsonApiMetricsConfig { * The minimum value that the meter is expected to observe for the command processor latency. The * unit is milliseconds. */ - @WithDefault("100") + @WithDefault("50") int MinExpectedCommandProcessorLatency(); /** diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java index 74fe8036bb..cb5c4d503e 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java @@ -333,7 +333,7 @@ public DistributionStatisticConfig configure( .percentilesHistogram(true) .minimumExpectedValue( TimeUnit.MILLISECONDS.toNanos( - jsonApiMetricsConfig.MinExpectedCommandProcessorLatency())) // 0.1 seconds + jsonApiMetricsConfig.MinExpectedCommandProcessorLatency())) // 0.05 seconds .maximumExpectedValue( TimeUnit.MILLISECONDS.toNanos( jsonApiMetricsConfig.MaxExpectedCommandProcessorLatency())) // 15 seconds