Skip to content
Open
5 changes: 4 additions & 1 deletion docs/content/config/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ When the same property is defined in multiple sources, the following precedence
| io.prometheus.metrics.summary_quantile_errors | [Summary.Builder.quantile(double, double)](</client_java/api/io/prometheus/metrics/core/metrics/Summary.Builder.html#quantile(double,double)>) | (5) |
| io.prometheus.metrics.summary_max_age_seconds | [Summary.Builder.maxAgeSeconds()](</client_java/api/io/prometheus/metrics/core/metrics/Summary.Builder.html#maxAgeSeconds(long)>) | |
| io.prometheus.metrics.summary_number_of_age_buckets | [Summary.Builder.numberOfAgeBuckets()](</client_java/api/io/prometheus/metrics/core/metrics/Summary.Builder.html#numberOfAgeBuckets(int)>) | |
| io.prometheus.metrics.use_otel_semconv | [MetricsProperties.useOtelSemconv()](</client_java/api/io/prometheus/metrics/config/MetricsProperties.html#useOtelSemconv()>) | (6) |

<!-- editorconfig-checker-enable -->

Expand All @@ -100,7 +101,9 @@ not just for counters<br>
(3) Comma-separated list. Example: `.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10`.<br>
(4) Comma-separated list. Example: `0.5, 0.95, 0.99`.<br>
(5) Comma-separated list. If specified, the list must have the same length as
`io.prometheus.metrics.summary_quantiles`. Example: `0.01, 0.005, 0.005`.
`io.prometheus.metrics.summary_quantiles`. Example: `0.01, 0.005, 0.005`.<br>
(6) Comma-separated list of OTel metric names. Use `*` to enable all.
Example: `jvm.gc.duration` or `*`.

There's one special feature about metric properties: You can set a property for one specific
metric only by specifying the metric name. Example:
Expand Down
23 changes: 23 additions & 0 deletions docs/content/instrumentation/jvm.md
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,29 @@ jvm_gc_collection_seconds_count{gc="PS Scavenge"} 0
jvm_gc_collection_seconds_sum{gc="PS Scavenge"} 0.0
```

For more detailed GC metrics, enable the
[use_otel_semconv](https://prometheus.github.io/client_java/config/config/#metrics-properties)
configuration option by specifying `jvm.gc.duration` or `*` (for all
OTel metrics). This replaces the standard metric with a histogram
implemented according to the
[OpenTelemetry Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/#metric-jvmgcduration). <!-- editorconfig-checker-disable-line -->

<!-- editorconfig-checker-disable -->

```text
# HELP jvm_gc_duration_seconds Duration of JVM garbage collection actions.
# TYPE jvm_gc_duration_seconds histogram
jvm_gc_duration_seconds_bucket{jvm_gc_action="end of minor GC",jvm_gc_name="G1 Young Generation",le="0.01"} 4
jvm_gc_duration_seconds_bucket{jvm_gc_action="end of minor GC",jvm_gc_name="G1 Young Generation",le="0.1"} 4
jvm_gc_duration_seconds_bucket{jvm_gc_action="end of minor GC",jvm_gc_name="G1 Young Generation",le="1.0"} 4
jvm_gc_duration_seconds_bucket{jvm_gc_action="end of minor GC",jvm_gc_name="G1 Young Generation",le="10.0"} 4
jvm_gc_duration_seconds_bucket{jvm_gc_action="end of minor GC",jvm_gc_name="G1 Young Generation",le="+Inf"} 4
jvm_gc_duration_seconds_count{jvm_gc_action="end of minor GC",jvm_gc_name="G1 Young Generation"} 4
jvm_gc_duration_seconds_sum{jvm_gc_action="end of minor GC",jvm_gc_name="G1 Young Generation"} 0.029
```

<!-- editorconfig-checker-enable -->

## JVM Memory Metrics

JVM memory metrics are provided by
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ public class MetricsProperties {
private static final String SUMMARY_QUANTILE_ERRORS = "summary_quantile_errors";
private static final String SUMMARY_MAX_AGE_SECONDS = "summary_max_age_seconds";
private static final String SUMMARY_NUMBER_OF_AGE_BUCKETS = "summary_number_of_age_buckets";
private static final String USE_OTEL_SEMCONV = "use_otel_semconv";

/**
* All known property suffixes that can be configured for metrics.
Expand All @@ -46,7 +47,8 @@ public class MetricsProperties {
SUMMARY_QUANTILES,
SUMMARY_QUANTILE_ERRORS,
SUMMARY_MAX_AGE_SECONDS,
SUMMARY_NUMBER_OF_AGE_BUCKETS
SUMMARY_NUMBER_OF_AGE_BUCKETS,
USE_OTEL_SEMCONV
};

@Nullable private final Boolean exemplarsEnabled;
Expand All @@ -62,6 +64,7 @@ public class MetricsProperties {
@Nullable private final List<Double> summaryQuantileErrors;
@Nullable private final Long summaryMaxAgeSeconds;
@Nullable private final Integer summaryNumberOfAgeBuckets;
@Nullable private final List<String> useOtelSemconv;

public MetricsProperties(
@Nullable Boolean exemplarsEnabled,
Expand Down Expand Up @@ -91,6 +94,7 @@ public MetricsProperties(
summaryQuantileErrors,
summaryMaxAgeSeconds,
summaryNumberOfAgeBuckets,
null,
"");
}

Expand All @@ -108,6 +112,7 @@ private MetricsProperties(
@Nullable List<Double> summaryQuantileErrors,
@Nullable Long summaryMaxAgeSeconds,
@Nullable Integer summaryNumberOfAgeBuckets,
@Nullable List<String> useOtelSemconv,
String configPropertyPrefix) {
this.exemplarsEnabled = exemplarsEnabled;
this.histogramNativeOnly = isHistogramNativeOnly(histogramClassicOnly, histogramNativeOnly);
Expand All @@ -129,6 +134,8 @@ private MetricsProperties(
: unmodifiableList(new ArrayList<>(summaryQuantileErrors));
this.summaryMaxAgeSeconds = summaryMaxAgeSeconds;
this.summaryNumberOfAgeBuckets = summaryNumberOfAgeBuckets;
this.useOtelSemconv =
useOtelSemconv == null ? null : unmodifiableList(new ArrayList<>(useOtelSemconv));
validate(configPropertyPrefix);
}

Expand Down Expand Up @@ -353,6 +360,15 @@ public Integer getSummaryNumberOfAgeBuckets() {
return summaryNumberOfAgeBuckets;
}

/**
* List of OTel metric names for which OpenTelemetry Semantic Conventions should be used. Use
* {@code "*"} to enable for all metrics. Returns {@code null} if not configured.
*/
@Nullable
public List<String> useOtelSemconv() {
return useOtelSemconv;
}

/**
* Note that this will remove entries from {@code propertySource}. This is because we want to know
* if there are unused properties remaining after all properties have been loaded.
Expand All @@ -373,6 +389,7 @@ static MetricsProperties load(String prefix, PropertySource propertySource)
Util.loadDoubleList(prefix, SUMMARY_QUANTILE_ERRORS, propertySource),
Util.loadLong(prefix, SUMMARY_MAX_AGE_SECONDS, propertySource),
Util.loadInteger(prefix, SUMMARY_NUMBER_OF_AGE_BUCKETS, propertySource),
Util.loadStringList(prefix, USE_OTEL_SEMCONV, propertySource),
prefix);
}

Expand All @@ -394,6 +411,7 @@ public static class Builder {
@Nullable private List<Double> summaryQuantileErrors;
@Nullable private Long summaryMaxAgeSeconds;
@Nullable private Integer summaryNumberOfAgeBuckets;
@Nullable private List<String> useOtelSemconv;

private Builder() {}

Expand All @@ -411,7 +429,9 @@ public MetricsProperties build() {
summaryQuantiles,
summaryQuantileErrors,
summaryMaxAgeSeconds,
summaryNumberOfAgeBuckets);
summaryNumberOfAgeBuckets,
useOtelSemconv,
"");
}

/** See {@link MetricsProperties#getExemplarsEnabled()} */
Expand Down Expand Up @@ -495,5 +515,11 @@ public Builder summaryNumberOfAgeBuckets(@Nullable Integer summaryNumberOfAgeBuc
this.summaryNumberOfAgeBuckets = summaryNumberOfAgeBuckets;
return this;
}

/** See {@link MetricsProperties#useOtelSemconv()} */
public Builder useOtelSemconv(String... useOtelSemconv) {
this.useOtelSemconv = Util.toStringList(useOtelSemconv);
return this;
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.prometheus.metrics.config;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;

Expand Down Expand Up @@ -167,6 +168,17 @@ public ExporterOpenTelemetryProperties getExporterOpenTelemetryProperties() {
return exporterOpenTelemetryProperties;
}

public boolean useOtelSemconv(String otelMetric) {
List<String> list = getDefaultMetricProperties().useOtelSemconv();
if (list == null || list.isEmpty()) {
return false;
}
if (list.contains("*")) {
return true;
}
return list.contains(otelMetric);
}

public static class Builder {
private MetricsProperties defaultMetricsProperties = MetricsProperties.builder().build();
private final MetricPropertiesMap metricProperties = new MetricPropertiesMap();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,14 @@ static Boolean loadBoolean(String prefix, String propertyName, PropertySource pr
return null;
}

@Nullable
static List<String> toStringList(@Nullable String... values) {
if (values == null) {
return null;
}
return Arrays.asList(values);
}

@Nullable
static List<Double> toList(@Nullable double... values) {
if (values == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,4 +128,36 @@ void testMetricNameStartingWithNumber() {
assertThat(result.getMetricProperties("123metric")).isSameAs(customProps);
assertThat(result.getMetricProperties("_23metric")).isSameAs(customProps);
}

@Test
void useOtelSemconvReturnsFalseForMetricNotInList() {
PrometheusProperties props = buildProperties("jvm.gc.duration");
assertThat(props.useOtelSemconv("other.metric")).isFalse();
}

@Test
void useOtelSemconvWildcardEnablesAll() {
PrometheusProperties props = buildProperties("*");
assertThat(props.useOtelSemconv("any.metric")).isTrue();
}

@Test
void useOtelSemconvNullListReturnsFalse() {
PrometheusProperties props = PrometheusProperties.get();
assertThat(props.useOtelSemconv("otel_y")).isFalse();
}

@Test
void useOtelSemconvSpecificMetricReturnsTrueForMatch() {
PrometheusProperties props = buildProperties("jvm.gc.duration", "jvm.memory.used");
assertThat(props.useOtelSemconv("jvm.gc.duration")).isTrue();
assertThat(props.useOtelSemconv("jvm.memory.used")).isTrue();
assertThat(props.useOtelSemconv("other.metric")).isFalse();
}

private static PrometheusProperties buildProperties(String... otelSemconv) {
return PrometheusProperties.builder()
.defaultMetricsProperties(MetricsProperties.builder().useOtelSemconv(otelSemconv).build())
.build();
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package io.prometheus.metrics.instrumentation.jvm;

import com.sun.management.GarbageCollectionNotificationInfo;
import io.prometheus.metrics.config.PrometheusProperties;
import io.prometheus.metrics.core.metrics.Histogram;
import io.prometheus.metrics.core.metrics.SummaryWithCallback;
import io.prometheus.metrics.model.registry.PrometheusRegistry;
import io.prometheus.metrics.model.snapshots.Labels;
Expand All @@ -10,6 +12,8 @@
import java.lang.management.ManagementFactory;
import java.util.List;
import javax.annotation.Nullable;
import javax.management.NotificationEmitter;
import javax.management.openmbean.CompositeData;

/**
* JVM Garbage Collector metrics. The {@link JvmGarbageCollectorMetrics} are registered as part of
Expand All @@ -19,14 +23,14 @@
* JvmMetrics.builder().register();
* }</pre>
*
* However, if you want only the {@link JvmGarbageCollectorMetrics} you can also register them
* <p>However, if you want only the {@link JvmGarbageCollectorMetrics} you can also register them
* directly:
*
* <pre>{@code
* JvmGarbageCollectorMetrics.builder().register();
* }</pre>
*
* Example metrics being exported:
* <p>Example metrics being exported:
*
* <pre>
* # HELP jvm_gc_collection_seconds Time spent in a given JVM garbage collector in seconds.
Expand All @@ -40,6 +44,7 @@
public class JvmGarbageCollectorMetrics {

private static final String JVM_GC_COLLECTION_SECONDS = "jvm_gc_collection_seconds";
private static final String JVM_GC_DURATION = "jvm.gc.duration";
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we be consistent with the way the other metric is defined?

Suggested change
private static final String JVM_GC_DURATION = "jvm.gc.duration";
private static final String JVM_GC_DURATION = "jvm_gc_duration";


private final PrometheusProperties config;
private final List<GarbageCollectorMXBean> garbageCollectorBeans;
Expand All @@ -55,7 +60,14 @@ private JvmGarbageCollectorMetrics(
}

private void register(PrometheusRegistry registry) {
if (config.useOtelSemconv(JVM_GC_DURATION)) {
registerOtel(registry);
} else {
registerPrometheus(registry);
}
}

private void registerPrometheus(PrometheusRegistry registry) {
SummaryWithCallback.builder(config)
.name(JVM_GC_COLLECTION_SECONDS)
.help("Time spent in a given JVM garbage collector in seconds.")
Expand All @@ -75,6 +87,54 @@ private void register(PrometheusRegistry registry) {
.register(registry);
}

private void registerOtel(PrometheusRegistry registry) {
double[] buckets = {0.01, 0.1, 1, 10};

Histogram gcDurationHistogram =
Histogram.builder(config)
.name(JVM_GC_DURATION)
.unit(Unit.SECONDS)
.help("Duration of JVM garbage collection actions.")
.labelNames("jvm.gc.action", "jvm.gc.name", "jvm.gc.cause")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

jvm.gc.cause is still experimental, should we hold off on that one until it stabilizes?

.classicUpperBounds(buckets)
.register(registry);

registerNotificationListener(gcDurationHistogram);
}

private void registerNotificationListener(Histogram gcDurationHistogram) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

taking a look at the OTel implementation, they also track the listeners and do a cleanup, not sure if it makes sense for us to implement something like that, by making JvmMetrics an AutoClosable. Looks like it ends up requiring a lot of changes, not sure if its worth it

Another thing they do is check to see if the class exists first , which I think would be good to do here too

for (GarbageCollectorMXBean gcBean : garbageCollectorBeans) {

if (!(gcBean instanceof NotificationEmitter)) {
continue;
}

((NotificationEmitter) gcBean)
.addNotificationListener(
(notification, handback) -> {
if (!GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION.equals(
notification.getType())) {
return;
}

GarbageCollectionNotificationInfo info =
GarbageCollectionNotificationInfo.from(
(CompositeData) notification.getUserData());

observe(gcDurationHistogram, info);
Comment on lines +115 to +124
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

try/catch here to avoid crashing the JVM is the notification listener throws something

Suggested change
if (!GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION.equals(
notification.getType())) {
return;
}
GarbageCollectionNotificationInfo info =
GarbageCollectionNotificationInfo.from(
(CompositeData) notification.getUserData());
observe(gcDurationHistogram, info);
try {
if (!GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION.equals(
notification.getType())) {
return;
}
GarbageCollectionNotificationInfo info =
GarbageCollectionNotificationInfo.from(
(CompositeData) notification.getUserData());
observe(gcDurationHistogram, info);
} catch (Exception e) {
logger.warning(
"Exception while processing garbage collection notification: " + e.getMessage());
}

},
null,
null);
}
}

private void observe(Histogram gcDurationHistogram, GarbageCollectionNotificationInfo info) {
double observedDuration = Unit.millisToSeconds(info.getGcInfo().getDuration());
gcDurationHistogram
.labelValues(info.getGcAction(), info.getGcName(), info.getGcCause())
.observe(observedDuration);
}

public static Builder builder() {
return new Builder(PrometheusProperties.get());
}
Expand Down
Loading