From 2ff9d1e8bd50b6f6e3f51975111c38927564b302 Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Fri, 27 Mar 2026 12:10:00 -0400 Subject: [PATCH 01/15] POC: Send JVM runtime metrics via OTLP using OTel-native naming Adds jvm.memory.used, jvm.memory.committed, jvm.memory.limit, jvm.gc.duration, jvm.gc.count, jvm.thread.count, jvm.class.loaded, jvm.class.unloaded, jvm.cpu.recent_utilization, jvm.cpu.count as OTel instruments on the existing OTLP metrics pipeline. Includes jvm.memory.type attribute for heap/non_heap breakdown required by semantic-core equivalence mappings. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../shim/metrics/JvmOtlpRuntimeMetrics.java | 243 ++++++++++++++++++ .../metrics/JvmOtlpRuntimeMetricsTest.groovy | 139 ++++++++++ 2 files changed, 382 insertions(+) create mode 100644 dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java create mode 100644 dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java new file mode 100644 index 00000000000..68120f72159 --- /dev/null +++ b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java @@ -0,0 +1,243 @@ +package datadog.opentelemetry.shim.metrics; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.Meter; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.ThreadMXBean; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Registers JVM runtime metrics using OTel semantic convention names via the dd-trace-java OTLP + * metrics pipeline. These metrics flow via OTLP without requiring a Datadog Agent or DogStatsD. + * + *

OTel JVM runtime metrics conventions: + * https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ + * + *

Semantic-core equivalence mappings: + * https://github.com/DataDog/semantic-core/blob/main/sor/domains/metrics/integrations/java/_equivalence/ + */ +public final class JvmOtlpRuntimeMetrics { + + private static final Logger log = LoggerFactory.getLogger(JvmOtlpRuntimeMetrics.class); + private static final String INSTRUMENTATION_SCOPE = "datadog.jvm.runtime"; + + private static volatile boolean started = false; + + /** Registers all JVM runtime metric instruments on the OTel MeterProvider. */ + public static void start() { + if (started) { + return; + } + started = true; + + try { + Meter meter = OtelMeterProvider.INSTANCE.get(INSTRUMENTATION_SCOPE); + registerMemoryMetrics(meter); + registerGcMetrics(meter); + registerThreadMetrics(meter); + registerClassLoadingMetrics(meter); + registerCpuMetrics(meter); + log.debug("Started OTLP runtime metrics with OTel-native naming (jvm.*)"); + } catch (Exception e) { + log.error("Failed to start JVM OTLP runtime metrics", e); + } + } + + /** + * jvm.memory.used - JVM memory used, split by type (heap/non_heap) and pool. + * + *

Maps to: jvm.heap_memory, jvm.non_heap_memory (via semantic-core, requires Sum By) + */ + private static void registerMemoryMetrics(Meter meter) { + MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); + List pools = ManagementFactory.getMemoryPoolMXBeans(); + + // jvm.memory.used - Measure of memory used + meter + .upDownCounterBuilder("jvm.memory.used") + .setDescription("Measure of memory used.") + .setUnit("By") + .buildWithCallback( + measurement -> { + // Heap total + measurement.record( + memoryBean.getHeapMemoryUsage().getUsed(), + Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); + // Non-heap total + measurement.record( + memoryBean.getNonHeapMemoryUsage().getUsed(), + Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "non_heap")); + // Per-pool breakdown + for (MemoryPoolMXBean pool : pools) { + measurement.record( + pool.getUsage().getUsed(), + Attributes.of( + AttributeKey.stringKey("jvm.memory.type"), + pool.getType().name().toLowerCase(), + AttributeKey.stringKey("jvm.memory.pool.name"), + pool.getName())); + } + }); + + // jvm.memory.committed - Measure of memory committed + meter + .upDownCounterBuilder("jvm.memory.committed") + .setDescription("Measure of memory committed.") + .setUnit("By") + .buildWithCallback( + measurement -> { + measurement.record( + memoryBean.getHeapMemoryUsage().getCommitted(), + Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); + measurement.record( + memoryBean.getNonHeapMemoryUsage().getCommitted(), + Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "non_heap")); + }); + + // jvm.memory.limit - Measure of max obtainable memory + meter + .upDownCounterBuilder("jvm.memory.limit") + .setDescription("Measure of max obtainable memory.") + .setUnit("By") + .buildWithCallback( + measurement -> { + long heapMax = memoryBean.getHeapMemoryUsage().getMax(); + if (heapMax > 0) { + measurement.record( + heapMax, + Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); + } + long nonHeapMax = memoryBean.getNonHeapMemoryUsage().getMax(); + if (nonHeapMax > 0) { + measurement.record( + nonHeapMax, + Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "non_heap")); + } + }); + } + + /** + * jvm.gc.duration - Duration of JVM garbage collection actions. Maps to: jvm.gc.pause_time (via + * semantic-core) + */ + private static void registerGcMetrics(Meter meter) { + List gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); + + // jvm.gc.duration - GC collection time (monotonic counter, in seconds) + meter + .counterBuilder("jvm.gc.duration") + .ofDoubles() + .setDescription("Duration of JVM garbage collection actions.") + .setUnit("s") + .buildWithCallback( + measurement -> { + for (GarbageCollectorMXBean gc : gcBeans) { + long timeMs = gc.getCollectionTime(); + if (timeMs >= 0) { + measurement.record( + timeMs / 1000.0, + Attributes.of( + AttributeKey.stringKey("jvm.gc.name"), + gc.getName(), + AttributeKey.stringKey("jvm.gc.action"), + gc.getName())); + } + } + }); + + // jvm.gc.count - Number of GC collections + meter + .counterBuilder("jvm.gc.count") + .setDescription("Number of executions of the garbage collector.") + .setUnit("{collection}") + .buildWithCallback( + measurement -> { + for (GarbageCollectorMXBean gc : gcBeans) { + long count = gc.getCollectionCount(); + if (count >= 0) { + measurement.record( + count, + Attributes.of(AttributeKey.stringKey("jvm.gc.name"), gc.getName())); + } + } + }); + } + + /** jvm.thread.count - Number of executing threads. Maps to: jvm.thread_count (via semantic-core) */ + private static void registerThreadMetrics(Meter meter) { + ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); + + meter + .upDownCounterBuilder("jvm.thread.count") + .setDescription("Number of executing platform threads.") + .setUnit("{thread}") + .buildWithCallback( + measurement -> { + measurement.record(threadBean.getThreadCount()); + }); + } + + /** jvm.class.loaded - Number of loaded classes. */ + private static void registerClassLoadingMetrics(Meter meter) { + meter + .upDownCounterBuilder("jvm.class.loaded") + .setDescription("Number of classes currently loaded.") + .setUnit("{class}") + .buildWithCallback( + measurement -> { + measurement.record( + ManagementFactory.getClassLoadingMXBean().getLoadedClassCount()); + }); + + meter + .counterBuilder("jvm.class.unloaded") + .setDescription("Number of classes unloaded since JVM start.") + .setUnit("{class}") + .buildWithCallback( + measurement -> { + measurement.record( + ManagementFactory.getClassLoadingMXBean().getUnloadedClassCount()); + }); + } + + /** jvm.cpu.recent_utilization - Recent CPU utilization by the JVM process. */ + private static void registerCpuMetrics(Meter meter) { + meter + .gaugeBuilder("jvm.cpu.recent_utilization") + .setDescription("Recent CPU utilization for the process as reported by the JVM.") + .setUnit("1") + .buildWithCallback( + measurement -> { + try { + java.lang.management.OperatingSystemMXBean osBean = + ManagementFactory.getOperatingSystemMXBean(); + if (osBean instanceof com.sun.management.OperatingSystemMXBean) { + double cpuLoad = + ((com.sun.management.OperatingSystemMXBean) osBean).getProcessCpuLoad(); + if (cpuLoad >= 0) { + measurement.record(cpuLoad); + } + } + } catch (Exception e) { + // com.sun.management may not be available on all JVMs + } + }); + + meter + .upDownCounterBuilder("jvm.cpu.count") + .setDescription("Number of processors available to the JVM.") + .setUnit("{cpu}") + .buildWithCallback( + measurement -> { + measurement.record(Runtime.getRuntime().availableProcessors()); + }); + } + + private JvmOtlpRuntimeMetrics() {} +} diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy new file mode 100644 index 00000000000..b3296620ef7 --- /dev/null +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy @@ -0,0 +1,139 @@ +package opentelemetry147.metrics + +import datadog.opentelemetry.shim.metrics.JvmOtlpRuntimeMetrics +import datadog.opentelemetry.shim.metrics.OtelMeterProvider +import datadog.trace.agent.test.InstrumentationSpecification +import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope +import datadog.trace.bootstrap.otel.metrics.OtelInstrumentDescriptor +import datadog.trace.bootstrap.otel.metrics.data.OtlpDoublePoint +import datadog.trace.bootstrap.otel.metrics.data.OtlpLongPoint +import datadog.trace.bootstrap.otel.metrics.data.OtelMetricRegistry +import datadog.trace.bootstrap.otel.metrics.data.OtlpDataPoint +import datadog.trace.bootstrap.otel.metrics.export.OtlpMetricVisitor +import datadog.trace.bootstrap.otel.metrics.export.OtlpMetricsVisitor +import datadog.trace.bootstrap.otel.metrics.export.OtlpScopedMetricsVisitor + +/** + * Tests that JVM runtime metrics are registered and exported via OTLP + * using OTel semantic convention names (jvm.memory.used, jvm.thread.count, etc.). + * + * Ref: https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ + * Ref: https://github.com/DataDog/semantic-core/blob/main/sor/domains/metrics/integrations/java/_equivalence/ + */ +class JvmOtlpRuntimeMetricsTest extends InstrumentationSpecification { + + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig("dd.metrics.otel.enabled", "true") + } + + def "JVM runtime metrics are registered and produce data points"() { + when: + JvmOtlpRuntimeMetrics.start() + def collector = new MetricCollector() + OtelMetricRegistry.INSTANCE.collectMetrics(collector) + + then: + // OtelInstrumentDescriptor.name is UTF8BytesString, convert to String for comparison + def names = collector.metricNames.collect { it.toString() } + "jvm.memory.used" in names + "jvm.memory.committed" in names + "jvm.thread.count" in names + "jvm.class.loaded" in names + "jvm.cpu.count" in names + "jvm.gc.duration" in names + "jvm.gc.count" in names + "jvm.class.unloaded" in names + "jvm.cpu.recent_utilization" in names + } + + def "jvm.memory.used has heap and non_heap type attributes"() { + when: + JvmOtlpRuntimeMetrics.start() + def collector = new MetricCollector() + OtelMetricRegistry.INSTANCE.collectMetrics(collector) + + then: + def types = collector.attributeValues("jvm.memory.used", "jvm.memory.type") + types.contains("heap") + types.contains("non_heap") + } + + def "jvm.memory.used heap value is positive"() { + when: + JvmOtlpRuntimeMetrics.start() + def collector = new MetricCollector() + OtelMetricRegistry.INSTANCE.collectMetrics(collector) + + then: + def heapPoints = collector.points["jvm.memory.used"] + .findAll { it.attrs["jvm.memory.type"] == "heap" } + heapPoints.size() > 0 + heapPoints[0].value > 0 + } + + def "jvm.thread.count is positive"() { + when: + JvmOtlpRuntimeMetrics.start() + def collector = new MetricCollector() + OtelMetricRegistry.INSTANCE.collectMetrics(collector) + + then: + def threadPoints = collector.points["jvm.thread.count"] + threadPoints.size() > 0 + threadPoints[0].value > 0 + } + + static class DataPointEntry { + Map attrs + Number value + } + + static class MetricCollector implements OtlpMetricsVisitor, OtlpScopedMetricsVisitor, OtlpMetricVisitor { + String currentInstrument = "" + Map currentAttrs = [:] + Set metricNames = new LinkedHashSet<>() + Map> points = [:].withDefault { [] } + + @Override + OtlpScopedMetricsVisitor visitScopedMetrics(OtelInstrumentationScope scope) { + return this + } + + @Override + OtlpMetricVisitor visitMetric(OtelInstrumentDescriptor descriptor) { + currentInstrument = descriptor.name.toString() + metricNames.add(descriptor.name.toString()) + return this + } + + @Override + void visitAttribute(int type, String key, Object value) { + currentAttrs.put(key.toString(), value.toString()) + } + + @Override + void visitDataPoint(OtlpDataPoint point) { + def attrs = new HashMap(currentAttrs) + currentAttrs.clear() + Number value = 0 + if (point instanceof OtlpLongPoint) { + value = ((OtlpLongPoint) point).value + } else if (point instanceof OtlpDoublePoint) { + value = ((OtlpDoublePoint) point).value + } + def entry = new DataPointEntry() + entry.attrs = attrs + entry.value = value + points[currentInstrument].add(entry) + } + + Set attributeValues(String metricName, String attrKey) { + points[metricName] + .collect { it.attrs[attrKey] } + .findAll { it != null } + .toSet() + } + } +} From cc5fa97515805e71702e1d9eb963dd5c47017df4 Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Fri, 27 Mar 2026 13:13:10 -0400 Subject: [PATCH 02/15] Add missing JVM metrics: jvm.memory.init, jvm.buffer.*, jvm.system.cpu.utilization, jvm.class.count Aligns with OTel JVM semantic conventions spreadsheet. Updates test to verify all 16 metrics are registered. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../shim/metrics/JvmOtlpRuntimeMetrics.java | 140 +++++++++++++++++- .../metrics/JvmOtlpRuntimeMetricsTest.groovy | 19 ++- 2 files changed, 155 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java index 68120f72159..72fd6d4a6a1 100644 --- a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java +++ b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java @@ -3,6 +3,7 @@ import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.Meter; +import java.lang.management.BufferPoolMXBean; import java.lang.management.GarbageCollectorMXBean; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; @@ -39,6 +40,7 @@ public static void start() { try { Meter meter = OtelMeterProvider.INSTANCE.get(INSTRUMENTATION_SCOPE); registerMemoryMetrics(meter); + registerBufferMetrics(meter); registerGcMetrics(meter); registerThreadMetrics(meter); registerClassLoadingMetrics(meter); @@ -98,6 +100,15 @@ private static void registerMemoryMetrics(Meter meter) { measurement.record( memoryBean.getNonHeapMemoryUsage().getCommitted(), Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "non_heap")); + for (MemoryPoolMXBean pool : pools) { + measurement.record( + pool.getUsage().getCommitted(), + Attributes.of( + AttributeKey.stringKey("jvm.memory.type"), + pool.getType().name().toLowerCase(), + AttributeKey.stringKey("jvm.memory.pool.name"), + pool.getName())); + } }); // jvm.memory.limit - Measure of max obtainable memory @@ -119,6 +130,101 @@ private static void registerMemoryMetrics(Meter meter) { nonHeapMax, Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "non_heap")); } + for (MemoryPoolMXBean pool : pools) { + long max = pool.getUsage().getMax(); + if (max > 0) { + measurement.record( + max, + Attributes.of( + AttributeKey.stringKey("jvm.memory.type"), + pool.getType().name().toLowerCase(), + AttributeKey.stringKey("jvm.memory.pool.name"), + pool.getName())); + } + } + }); + + // jvm.memory.init - Measure of initial memory requested + meter + .upDownCounterBuilder("jvm.memory.init") + .setDescription("Measure of initial memory requested.") + .setUnit("By") + .buildWithCallback( + measurement -> { + long heapInit = memoryBean.getHeapMemoryUsage().getInit(); + if (heapInit > 0) { + measurement.record( + heapInit, + Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); + } + long nonHeapInit = memoryBean.getNonHeapMemoryUsage().getInit(); + if (nonHeapInit > 0) { + measurement.record( + nonHeapInit, + Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "non_heap")); + } + }); + } + + /** + * jvm.buffer.* - JVM buffer pool metrics (direct, mapped). Maps to: jvm.buffer_pool.* (via + * semantic-core) + */ + private static void registerBufferMetrics(Meter meter) { + List bufferPools = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); + + // jvm.buffer.memory.used + meter + .upDownCounterBuilder("jvm.buffer.memory.used") + .setDescription("Measure of memory used by buffers.") + .setUnit("By") + .buildWithCallback( + measurement -> { + for (BufferPoolMXBean pool : bufferPools) { + long used = pool.getMemoryUsed(); + if (used >= 0) { + measurement.record( + used, + Attributes.of( + AttributeKey.stringKey("jvm.buffer.pool.name"), pool.getName())); + } + } + }); + + // jvm.buffer.memory.limit + meter + .upDownCounterBuilder("jvm.buffer.memory.limit") + .setDescription("Measure of total memory capacity of buffers.") + .setUnit("By") + .buildWithCallback( + measurement -> { + for (BufferPoolMXBean pool : bufferPools) { + long limit = pool.getTotalCapacity(); + if (limit >= 0) { + measurement.record( + limit, + Attributes.of( + AttributeKey.stringKey("jvm.buffer.pool.name"), pool.getName())); + } + } + }); + + // jvm.buffer.count + meter + .upDownCounterBuilder("jvm.buffer.count") + .setDescription("Number of buffers in the pool.") + .setUnit("{buffer}") + .buildWithCallback( + measurement -> { + for (BufferPoolMXBean pool : bufferPools) { + long count = pool.getCount(); + if (count >= 0) { + measurement.record( + count, + Attributes.of( + AttributeKey.stringKey("jvm.buffer.pool.name"), pool.getName())); + } + } }); } @@ -183,10 +289,20 @@ private static void registerThreadMetrics(Meter meter) { }); } - /** jvm.class.loaded - Number of loaded classes. */ + /** jvm.class.* - Class loading metrics. */ private static void registerClassLoadingMetrics(Meter meter) { meter .upDownCounterBuilder("jvm.class.loaded") + .setDescription("Number of classes loaded since JVM start.") + .setUnit("{class}") + .buildWithCallback( + measurement -> { + measurement.record( + ManagementFactory.getClassLoadingMXBean().getTotalLoadedClassCount()); + }); + + meter + .upDownCounterBuilder("jvm.class.count") .setDescription("Number of classes currently loaded.") .setUnit("{class}") .buildWithCallback( @@ -237,6 +353,28 @@ private static void registerCpuMetrics(Meter meter) { measurement -> { measurement.record(Runtime.getRuntime().availableProcessors()); }); + + // jvm.system.cpu.utilization - Recent CPU utilization for the whole system + meter + .gaugeBuilder("jvm.system.cpu.utilization") + .setDescription("Recent CPU utilization for the whole system as reported by the JVM.") + .setUnit("1") + .buildWithCallback( + measurement -> { + try { + java.lang.management.OperatingSystemMXBean osBean = + ManagementFactory.getOperatingSystemMXBean(); + if (osBean instanceof com.sun.management.OperatingSystemMXBean) { + double load = + ((com.sun.management.OperatingSystemMXBean) osBean).getSystemCpuLoad(); + if (load >= 0) { + measurement.record(load); + } + } + } catch (Exception e) { + // com.sun.management may not be available + } + }); } private JvmOtlpRuntimeMetrics() {} diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy index b3296620ef7..a9d8dfb80ff 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy @@ -37,15 +37,28 @@ class JvmOtlpRuntimeMetricsTest extends InstrumentationSpecification { then: // OtelInstrumentDescriptor.name is UTF8BytesString, convert to String for comparison def names = collector.metricNames.collect { it.toString() } + // Memory "jvm.memory.used" in names "jvm.memory.committed" in names - "jvm.thread.count" in names - "jvm.class.loaded" in names - "jvm.cpu.count" in names + "jvm.memory.limit" in names + "jvm.memory.init" in names + // Buffers + "jvm.buffer.memory.used" in names + "jvm.buffer.memory.limit" in names + "jvm.buffer.count" in names + // GC "jvm.gc.duration" in names "jvm.gc.count" in names + // Threads + "jvm.thread.count" in names + // Classes + "jvm.class.loaded" in names + "jvm.class.count" in names "jvm.class.unloaded" in names + // CPU + "jvm.cpu.count" in names "jvm.cpu.recent_utilization" in names + "jvm.system.cpu.utilization" in names } def "jvm.memory.used has heap and non_heap type attributes"() { From 441d7108a08493d30e861a95073eac3f438b31d0 Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Thu, 2 Apr 2026 14:14:06 -0400 Subject: [PATCH 03/15] Wire JvmOtlpRuntimeMetrics into agent startup Call JvmOtlpRuntimeMetrics.start() from OpenTelemetryMetricsInstrumentation when both DD_METRICS_OTEL_ENABLED and DD_RUNTIME_METRICS_ENABLED are true. Matches .NET/Go/NodeJS config gating pattern. Also updates metrics to match OTel spec exactly (Option B): removes jvm.gc.duration/count, fixes types, adds missing metrics. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../shim/metrics/JvmOtlpRuntimeMetrics.java | 243 +++++++++++------- .../OpenTelemetryMetricsInstrumentation.java | 7 + .../metrics/JvmOtlpRuntimeMetricsTest.groovy | 17 +- 3 files changed, 162 insertions(+), 105 deletions(-) diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java index 72fd6d4a6a1..5cb93f4c3f3 100644 --- a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java +++ b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java @@ -4,10 +4,10 @@ import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.Meter; import java.lang.management.BufferPoolMXBean; -import java.lang.management.GarbageCollectorMXBean; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryUsage; import java.lang.management.ThreadMXBean; import java.util.List; import org.slf4j.Logger; @@ -17,6 +17,9 @@ * Registers JVM runtime metrics using OTel semantic convention names via the dd-trace-java OTLP * metrics pipeline. These metrics flow via OTLP without requiring a Datadog Agent or DogStatsD. * + *

Only includes metrics where we can match the exact OTel spec type. Metrics requiring Histogram + * type (jvm.gc.duration) are excluded because JMX cannot produce distribution data. + * *

OTel JVM runtime metrics conventions: * https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ * @@ -41,41 +44,41 @@ public static void start() { Meter meter = OtelMeterProvider.INSTANCE.get(INSTRUMENTATION_SCOPE); registerMemoryMetrics(meter); registerBufferMetrics(meter); - registerGcMetrics(meter); registerThreadMetrics(meter); registerClassLoadingMetrics(meter); registerCpuMetrics(meter); + registerFileDescriptorMetrics(meter); log.debug("Started OTLP runtime metrics with OTel-native naming (jvm.*)"); } catch (Exception e) { log.error("Failed to start JVM OTLP runtime metrics", e); } } + // Note: jvm.gc.duration is excluded — OTel spec requires Histogram type but JMX only provides + // cumulative milliseconds via GarbageCollectorMXBean.getCollectionTime(), not individual + // GC event durations needed to build a distribution. + /** - * jvm.memory.used - JVM memory used, split by type (heap/non_heap) and pool. - * - *

Maps to: jvm.heap_memory, jvm.non_heap_memory (via semantic-core, requires Sum By) + * jvm.memory.used, jvm.memory.committed, jvm.memory.limit, jvm.memory.init, + * jvm.memory.used_after_last_gc — all UpDownCounter per spec. */ private static void registerMemoryMetrics(Meter meter) { MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); List pools = ManagementFactory.getMemoryPoolMXBeans(); - // jvm.memory.used - Measure of memory used + // jvm.memory.used (UpDownCounter, Stable) meter .upDownCounterBuilder("jvm.memory.used") .setDescription("Measure of memory used.") .setUnit("By") .buildWithCallback( measurement -> { - // Heap total measurement.record( memoryBean.getHeapMemoryUsage().getUsed(), Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); - // Non-heap total measurement.record( memoryBean.getNonHeapMemoryUsage().getUsed(), Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "non_heap")); - // Per-pool breakdown for (MemoryPoolMXBean pool : pools) { measurement.record( pool.getUsage().getUsed(), @@ -87,7 +90,7 @@ private static void registerMemoryMetrics(Meter meter) { } }); - // jvm.memory.committed - Measure of memory committed + // jvm.memory.committed (UpDownCounter, Stable) meter .upDownCounterBuilder("jvm.memory.committed") .setDescription("Measure of memory committed.") @@ -111,7 +114,7 @@ private static void registerMemoryMetrics(Meter meter) { } }); - // jvm.memory.limit - Measure of max obtainable memory + // jvm.memory.limit (UpDownCounter, Stable) meter .upDownCounterBuilder("jvm.memory.limit") .setDescription("Measure of max obtainable memory.") @@ -144,7 +147,7 @@ private static void registerMemoryMetrics(Meter meter) { } }); - // jvm.memory.init - Measure of initial memory requested + // jvm.memory.init (UpDownCounter, Development) meter .upDownCounterBuilder("jvm.memory.init") .setDescription("Measure of initial memory requested.") @@ -164,16 +167,37 @@ private static void registerMemoryMetrics(Meter meter) { Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "non_heap")); } }); + + // jvm.memory.used_after_last_gc (UpDownCounter, Stable) + meter + .upDownCounterBuilder("jvm.memory.used_after_last_gc") + .setDescription("Measure of memory used after the most recent garbage collection event.") + .setUnit("By") + .buildWithCallback( + measurement -> { + for (MemoryPoolMXBean pool : pools) { + MemoryUsage collectionUsage = pool.getCollectionUsage(); + if (collectionUsage != null) { + long used = collectionUsage.getUsed(); + if (used >= 0) { + measurement.record( + used, + Attributes.of( + AttributeKey.stringKey("jvm.memory.type"), + pool.getType().name().toLowerCase(), + AttributeKey.stringKey("jvm.memory.pool.name"), + pool.getName())); + } + } + } + }); } - /** - * jvm.buffer.* - JVM buffer pool metrics (direct, mapped). Maps to: jvm.buffer_pool.* (via - * semantic-core) - */ + /** jvm.buffer.* (UpDownCounter, Development) — JVM buffer pool metrics (direct, mapped). */ private static void registerBufferMetrics(Meter meter) { - List bufferPools = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); + List bufferPools = + ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); - // jvm.buffer.memory.used meter .upDownCounterBuilder("jvm.buffer.memory.used") .setDescription("Measure of memory used by buffers.") @@ -191,7 +215,6 @@ private static void registerBufferMetrics(Meter meter) { } }); - // jvm.buffer.memory.limit meter .upDownCounterBuilder("jvm.buffer.memory.limit") .setDescription("Measure of total memory capacity of buffers.") @@ -209,7 +232,6 @@ private static void registerBufferMetrics(Meter meter) { } }); - // jvm.buffer.count meter .upDownCounterBuilder("jvm.buffer.count") .setDescription("Number of buffers in the pool.") @@ -228,54 +250,7 @@ private static void registerBufferMetrics(Meter meter) { }); } - /** - * jvm.gc.duration - Duration of JVM garbage collection actions. Maps to: jvm.gc.pause_time (via - * semantic-core) - */ - private static void registerGcMetrics(Meter meter) { - List gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); - - // jvm.gc.duration - GC collection time (monotonic counter, in seconds) - meter - .counterBuilder("jvm.gc.duration") - .ofDoubles() - .setDescription("Duration of JVM garbage collection actions.") - .setUnit("s") - .buildWithCallback( - measurement -> { - for (GarbageCollectorMXBean gc : gcBeans) { - long timeMs = gc.getCollectionTime(); - if (timeMs >= 0) { - measurement.record( - timeMs / 1000.0, - Attributes.of( - AttributeKey.stringKey("jvm.gc.name"), - gc.getName(), - AttributeKey.stringKey("jvm.gc.action"), - gc.getName())); - } - } - }); - - // jvm.gc.count - Number of GC collections - meter - .counterBuilder("jvm.gc.count") - .setDescription("Number of executions of the garbage collector.") - .setUnit("{collection}") - .buildWithCallback( - measurement -> { - for (GarbageCollectorMXBean gc : gcBeans) { - long count = gc.getCollectionCount(); - if (count >= 0) { - measurement.record( - count, - Attributes.of(AttributeKey.stringKey("jvm.gc.name"), gc.getName())); - } - } - }); - } - - /** jvm.thread.count - Number of executing threads. Maps to: jvm.thread_count (via semantic-core) */ + /** jvm.thread.count (UpDownCounter, Stable) */ private static void registerThreadMetrics(Meter meter) { ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); @@ -283,78 +258,105 @@ private static void registerThreadMetrics(Meter meter) { .upDownCounterBuilder("jvm.thread.count") .setDescription("Number of executing platform threads.") .setUnit("{thread}") - .buildWithCallback( - measurement -> { - measurement.record(threadBean.getThreadCount()); - }); + .buildWithCallback(measurement -> measurement.record(threadBean.getThreadCount())); } - /** jvm.class.* - Class loading metrics. */ + /** + * jvm.class.loaded (Counter, Stable) — cumulative total loaded since JVM start. + * jvm.class.unloaded (Counter, Stable) — cumulative total unloaded since JVM start. + * jvm.class.count (UpDownCounter, Stable) — currently loaded count. + */ private static void registerClassLoadingMetrics(Meter meter) { + // jvm.class.loaded — Counter per spec (cumulative total, only goes up) meter - .upDownCounterBuilder("jvm.class.loaded") + .counterBuilder("jvm.class.loaded") .setDescription("Number of classes loaded since JVM start.") .setUnit("{class}") .buildWithCallback( - measurement -> { - measurement.record( - ManagementFactory.getClassLoadingMXBean().getTotalLoadedClassCount()); - }); + measurement -> + measurement.record( + ManagementFactory.getClassLoadingMXBean().getTotalLoadedClassCount())); + // jvm.class.count — UpDownCounter per spec (current count, can decrease) meter .upDownCounterBuilder("jvm.class.count") .setDescription("Number of classes currently loaded.") .setUnit("{class}") .buildWithCallback( - measurement -> { - measurement.record( - ManagementFactory.getClassLoadingMXBean().getLoadedClassCount()); - }); + measurement -> + measurement.record( + ManagementFactory.getClassLoadingMXBean().getLoadedClassCount())); + // jvm.class.unloaded — Counter per spec meter .counterBuilder("jvm.class.unloaded") .setDescription("Number of classes unloaded since JVM start.") .setUnit("{class}") .buildWithCallback( - measurement -> { - measurement.record( - ManagementFactory.getClassLoadingMXBean().getUnloadedClassCount()); - }); + measurement -> + measurement.record( + ManagementFactory.getClassLoadingMXBean().getUnloadedClassCount())); } - /** jvm.cpu.recent_utilization - Recent CPU utilization by the JVM process. */ + /** + * jvm.cpu.time (Counter, Stable), jvm.cpu.count (UpDownCounter, Stable), + * jvm.cpu.recent_utilization (Gauge, Stable), jvm.system.cpu.utilization (Gauge, Development). + */ private static void registerCpuMetrics(Meter meter) { + // jvm.cpu.time — Counter per spec (cumulative CPU time in seconds) meter - .gaugeBuilder("jvm.cpu.recent_utilization") - .setDescription("Recent CPU utilization for the process as reported by the JVM.") - .setUnit("1") + .counterBuilder("jvm.cpu.time") + .ofDoubles() + .setDescription("CPU time used by the process as reported by the JVM.") + .setUnit("s") .buildWithCallback( measurement -> { try { java.lang.management.OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); if (osBean instanceof com.sun.management.OperatingSystemMXBean) { - double cpuLoad = - ((com.sun.management.OperatingSystemMXBean) osBean).getProcessCpuLoad(); - if (cpuLoad >= 0) { - measurement.record(cpuLoad); + long nanos = + ((com.sun.management.OperatingSystemMXBean) osBean).getProcessCpuTime(); + if (nanos >= 0) { + measurement.record(nanos / 1e9); } } } catch (Exception e) { - // com.sun.management may not be available on all JVMs + // com.sun.management may not be available } }); + // jvm.cpu.count — UpDownCounter per spec meter .upDownCounterBuilder("jvm.cpu.count") .setDescription("Number of processors available to the JVM.") .setUnit("{cpu}") + .buildWithCallback( + measurement -> measurement.record(Runtime.getRuntime().availableProcessors())); + + // jvm.cpu.recent_utilization — Gauge per spec + meter + .gaugeBuilder("jvm.cpu.recent_utilization") + .setDescription("Recent CPU utilization for the process as reported by the JVM.") + .setUnit("1") .buildWithCallback( measurement -> { - measurement.record(Runtime.getRuntime().availableProcessors()); + try { + java.lang.management.OperatingSystemMXBean osBean = + ManagementFactory.getOperatingSystemMXBean(); + if (osBean instanceof com.sun.management.OperatingSystemMXBean) { + double cpuLoad = + ((com.sun.management.OperatingSystemMXBean) osBean).getProcessCpuLoad(); + if (cpuLoad >= 0) { + measurement.record(cpuLoad); + } + } + } catch (Exception e) { + // com.sun.management may not be available + } }); - // jvm.system.cpu.utilization - Recent CPU utilization for the whole system + // jvm.system.cpu.utilization — Gauge, Development meter .gaugeBuilder("jvm.system.cpu.utilization") .setDescription("Recent CPU utilization for the whole system as reported by the JVM.") @@ -377,5 +379,52 @@ private static void registerCpuMetrics(Meter meter) { }); } + /** jvm.file_descriptor.count and jvm.file_descriptor.limit (UpDownCounter, Development). */ + private static void registerFileDescriptorMetrics(Meter meter) { + meter + .upDownCounterBuilder("jvm.file_descriptor.count") + .setDescription("Number of open file descriptors.") + .setUnit("{file_descriptor}") + .buildWithCallback( + measurement -> { + try { + java.lang.management.OperatingSystemMXBean osBean = + ManagementFactory.getOperatingSystemMXBean(); + if (osBean instanceof com.sun.management.UnixOperatingSystemMXBean) { + long count = + ((com.sun.management.UnixOperatingSystemMXBean) osBean) + .getOpenFileDescriptorCount(); + if (count >= 0) { + measurement.record(count); + } + } + } catch (Exception e) { + // UnixOperatingSystemMXBean not available on Windows + } + }); + + meter + .upDownCounterBuilder("jvm.file_descriptor.limit") + .setDescription("Maximum number of open file descriptors allowed.") + .setUnit("{file_descriptor}") + .buildWithCallback( + measurement -> { + try { + java.lang.management.OperatingSystemMXBean osBean = + ManagementFactory.getOperatingSystemMXBean(); + if (osBean instanceof com.sun.management.UnixOperatingSystemMXBean) { + long limit = + ((com.sun.management.UnixOperatingSystemMXBean) osBean) + .getMaxFileDescriptorCount(); + if (limit >= 0) { + measurement.record(limit); + } + } + } catch (Exception e) { + // UnixOperatingSystemMXBean not available on Windows + } + }); + } + private JvmOtlpRuntimeMetrics() {} } diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/main/java/datadog/trace/instrumentation/opentelemetry147/OpenTelemetryMetricsInstrumentation.java b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/main/java/datadog/trace/instrumentation/opentelemetry147/OpenTelemetryMetricsInstrumentation.java index 4cb6fd79ede..b293687ad0f 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/main/java/datadog/trace/instrumentation/opentelemetry147/OpenTelemetryMetricsInstrumentation.java +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/main/java/datadog/trace/instrumentation/opentelemetry147/OpenTelemetryMetricsInstrumentation.java @@ -7,6 +7,7 @@ import static net.bytebuddy.matcher.ElementMatchers.takesNoArguments; import com.google.auto.service.AutoService; +import datadog.opentelemetry.shim.metrics.JvmOtlpRuntimeMetrics; import datadog.opentelemetry.shim.metrics.OtelMeterProvider; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; @@ -101,6 +102,12 @@ public static class MeterProviderAdvice { @Advice.OnMethodExit(suppress = Throwable.class) public static void returnProvider(@Advice.Return(readOnly = false) MeterProvider result) { result = OtelMeterProvider.INSTANCE; + // Start JVM runtime metrics when both DD_METRICS_OTEL_ENABLED and + // DD_RUNTIME_METRICS_ENABLED are true, matching the .NET/Go/NodeJS pattern. + // JvmOtlpRuntimeMetrics.start() is idempotent (checks a started flag internally). + if (datadog.trace.api.Config.get().isRuntimeMetricsEnabled()) { + JvmOtlpRuntimeMetrics.start(); + } } public static void muzzleCheck(DoubleGauge doubleGauge) { diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy index a9d8dfb80ff..36805766bf9 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy @@ -37,28 +37,29 @@ class JvmOtlpRuntimeMetricsTest extends InstrumentationSpecification { then: // OtelInstrumentDescriptor.name is UTF8BytesString, convert to String for comparison def names = collector.metricNames.collect { it.toString() } - // Memory + // Memory (5 metrics, all UpDownCounter per spec) "jvm.memory.used" in names "jvm.memory.committed" in names "jvm.memory.limit" in names "jvm.memory.init" in names - // Buffers + "jvm.memory.used_after_last_gc" in names + // Buffers (3 metrics, UpDownCounter per spec) "jvm.buffer.memory.used" in names "jvm.buffer.memory.limit" in names "jvm.buffer.count" in names - // GC - "jvm.gc.duration" in names - "jvm.gc.count" in names - // Threads + // Threads (1 metric, UpDownCounter per spec) "jvm.thread.count" in names - // Classes + // Classes (3 metrics: loaded/unloaded are Counter, count is UpDownCounter per spec) "jvm.class.loaded" in names "jvm.class.count" in names "jvm.class.unloaded" in names - // CPU + // CPU (4 metrics per spec) + "jvm.cpu.time" in names "jvm.cpu.count" in names "jvm.cpu.recent_utilization" in names "jvm.system.cpu.utilization" in names + // NOT included: jvm.gc.duration (spec requires Histogram, JMX can't produce it) + // NOT included: jvm.gc.count (not in OTel spec) } def "jvm.memory.used has heap and non_heap type attributes"() { From 485b506b5eb78dedde8161d39623524368c1b28d Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Thu, 9 Apr 2026 12:53:27 -0400 Subject: [PATCH 04/15] Rebase on master: fix imports for renamed OTLP classes OtlpDataPoint, OtlpLongPoint, OtlpDoublePoint, OtlpMetricVisitor, OtlpMetricsVisitor, OtlpScopedMetricsVisitor moved from datadog.trace.bootstrap.otel.metrics.data/export to datadog.trace.bootstrap.otlp.metrics after PR #11055 merge. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../metrics/JvmOtlpRuntimeMetricsTest.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy index 36805766bf9..d12112d2c70 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy @@ -5,13 +5,13 @@ import datadog.opentelemetry.shim.metrics.OtelMeterProvider import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope import datadog.trace.bootstrap.otel.metrics.OtelInstrumentDescriptor -import datadog.trace.bootstrap.otel.metrics.data.OtlpDoublePoint -import datadog.trace.bootstrap.otel.metrics.data.OtlpLongPoint import datadog.trace.bootstrap.otel.metrics.data.OtelMetricRegistry -import datadog.trace.bootstrap.otel.metrics.data.OtlpDataPoint -import datadog.trace.bootstrap.otel.metrics.export.OtlpMetricVisitor -import datadog.trace.bootstrap.otel.metrics.export.OtlpMetricsVisitor -import datadog.trace.bootstrap.otel.metrics.export.OtlpScopedMetricsVisitor +import datadog.trace.bootstrap.otlp.metrics.OtlpDataPoint +import datadog.trace.bootstrap.otlp.metrics.OtlpDoublePoint +import datadog.trace.bootstrap.otlp.metrics.OtlpLongPoint +import datadog.trace.bootstrap.otlp.metrics.OtlpMetricVisitor +import datadog.trace.bootstrap.otlp.metrics.OtlpMetricsVisitor +import datadog.trace.bootstrap.otlp.metrics.OtlpScopedMetricsVisitor /** * Tests that JVM runtime metrics are registered and exported via OTLP From 8d6c3257f0f4eae00a852c5ea6e85d328edfa8ad Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Wed, 22 Apr 2026 15:01:38 -0400 Subject: [PATCH 05/15] Improve OTLP runtime metrics test: exact count, full list, no DD names - Assert exactly 18 metrics (was missing jvm.file_descriptor.count/limit) - Assert no DD-proprietary names present - Matches .NET PR #8457 test pattern Co-Authored-By: Claude Opus 4.6 (1M context) --- .../metrics/JvmOtlpRuntimeMetricsTest.groovy | 63 +++++++++++-------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy index d12112d2c70..cc31740700b 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy @@ -28,38 +28,51 @@ class JvmOtlpRuntimeMetricsTest extends InstrumentationSpecification { injectSysConfig("dd.metrics.otel.enabled", "true") } - def "JVM runtime metrics are registered and produce data points"() { + def "registers exactly 18 OTel-named JVM runtime metrics"() { when: JvmOtlpRuntimeMetrics.start() def collector = new MetricCollector() OtelMetricRegistry.INSTANCE.collectMetrics(collector) then: - // OtelInstrumentDescriptor.name is UTF8BytesString, convert to String for comparison def names = collector.metricNames.collect { it.toString() } - // Memory (5 metrics, all UpDownCounter per spec) - "jvm.memory.used" in names - "jvm.memory.committed" in names - "jvm.memory.limit" in names - "jvm.memory.init" in names - "jvm.memory.used_after_last_gc" in names - // Buffers (3 metrics, UpDownCounter per spec) - "jvm.buffer.memory.used" in names - "jvm.buffer.memory.limit" in names - "jvm.buffer.count" in names - // Threads (1 metric, UpDownCounter per spec) - "jvm.thread.count" in names - // Classes (3 metrics: loaded/unloaded are Counter, count is UpDownCounter per spec) - "jvm.class.loaded" in names - "jvm.class.count" in names - "jvm.class.unloaded" in names - // CPU (4 metrics per spec) - "jvm.cpu.time" in names - "jvm.cpu.count" in names - "jvm.cpu.recent_utilization" in names - "jvm.system.cpu.utilization" in names - // NOT included: jvm.gc.duration (spec requires Histogram, JMX can't produce it) - // NOT included: jvm.gc.count (not in OTel spec) + + def expectedMetrics = [ + // Memory (5 metrics) + "jvm.memory.used", + "jvm.memory.committed", + "jvm.memory.limit", + "jvm.memory.init", + "jvm.memory.used_after_last_gc", + // Buffers (3 metrics) + "jvm.buffer.memory.used", + "jvm.buffer.memory.limit", + "jvm.buffer.count", + // Threads (1 metric) + "jvm.thread.count", + // Classes (3 metrics) + "jvm.class.loaded", + "jvm.class.count", + "jvm.class.unloaded", + // CPU (4 metrics) + "jvm.cpu.time", + "jvm.cpu.count", + "jvm.cpu.recent_utilization", + "jvm.system.cpu.utilization", + // File descriptors (2 metrics) + "jvm.file_descriptor.count", + "jvm.file_descriptor.limit", + ] + + for (metric in expectedMetrics) { + assert metric in names : "Expected metric '${metric}' not found. Got: ${names.sort()}" + } + + names.size() == 18 + + // No DD-proprietary names should be present + def ddNames = names.findAll { it.startsWith("jvm.heap_memory") || it.startsWith("jvm.thread_count") } + ddNames.isEmpty() } def "jvm.memory.used has heap and non_heap type attributes"() { From 036c24405fa2c0b687c45e6fdb4cf9cd6fbe3095 Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Wed, 6 May 2026 15:06:53 -0400 Subject: [PATCH 06/15] Remove Opt-In JVM metrics from POC Per OTel semconv, jvm.system.cpu.utilization, jvm.system.cpu.load_1m, jvm.file_descriptor.count and jvm.file_descriptor.limit are Opt-In: they MUST only be reported when the instrumentation is explicitly configured to do so. This POC currently has no DD_METRICS_OTEL_OPTIN_ENABLED gating, so emitting them would violate the spec. Removing the three we shipped (jvm.system.cpu.utilization, jvm.file_descriptor.count, jvm.file_descriptor.limit) brings the metric set down to the 15 Recommended JVM metrics. Adding opt-in support is tracked separately and will reintroduce these behind the env var. Reference: https://opentelemetry.io/docs/specs/semconv/general/metric-requirement-level/#opt-in --- .../shim/metrics/JvmOtlpRuntimeMetrics.java | 83 ++----------------- .../metrics/JvmOtlpRuntimeMetricsTest.groovy | 13 +-- 2 files changed, 15 insertions(+), 81 deletions(-) diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java index 5cb93f4c3f3..12bdb8007df 100644 --- a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java +++ b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java @@ -47,7 +47,6 @@ public static void start() { registerThreadMetrics(meter); registerClassLoadingMetrics(meter); registerCpuMetrics(meter); - registerFileDescriptorMetrics(meter); log.debug("Started OTLP runtime metrics with OTel-native naming (jvm.*)"); } catch (Exception e) { log.error("Failed to start JVM OTLP runtime metrics", e); @@ -124,8 +123,7 @@ private static void registerMemoryMetrics(Meter meter) { long heapMax = memoryBean.getHeapMemoryUsage().getMax(); if (heapMax > 0) { measurement.record( - heapMax, - Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); + heapMax, Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); } long nonHeapMax = memoryBean.getNonHeapMemoryUsage().getMax(); if (nonHeapMax > 0) { @@ -157,8 +155,7 @@ private static void registerMemoryMetrics(Meter meter) { long heapInit = memoryBean.getHeapMemoryUsage().getInit(); if (heapInit > 0) { measurement.record( - heapInit, - Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); + heapInit, Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); } long nonHeapInit = memoryBean.getNonHeapMemoryUsage().getInit(); if (nonHeapInit > 0) { @@ -300,7 +297,12 @@ private static void registerClassLoadingMetrics(Meter meter) { /** * jvm.cpu.time (Counter, Stable), jvm.cpu.count (UpDownCounter, Stable), - * jvm.cpu.recent_utilization (Gauge, Stable), jvm.system.cpu.utilization (Gauge, Development). + * jvm.cpu.recent_utilization (Gauge, Stable). + * + *

jvm.system.cpu.utilization, jvm.system.cpu.load_1m are Opt-In per OTel semconv and are + * intentionally not emitted; gating those behind a future DD_METRICS_OTEL_OPTIN_ENABLED flag is + * tracked separately. See + * https://opentelemetry.io/docs/specs/semconv/general/metric-requirement-level/#opt-in. */ private static void registerCpuMetrics(Meter meter) { // jvm.cpu.time — Counter per spec (cumulative CPU time in seconds) @@ -355,75 +357,6 @@ private static void registerCpuMetrics(Meter meter) { // com.sun.management may not be available } }); - - // jvm.system.cpu.utilization — Gauge, Development - meter - .gaugeBuilder("jvm.system.cpu.utilization") - .setDescription("Recent CPU utilization for the whole system as reported by the JVM.") - .setUnit("1") - .buildWithCallback( - measurement -> { - try { - java.lang.management.OperatingSystemMXBean osBean = - ManagementFactory.getOperatingSystemMXBean(); - if (osBean instanceof com.sun.management.OperatingSystemMXBean) { - double load = - ((com.sun.management.OperatingSystemMXBean) osBean).getSystemCpuLoad(); - if (load >= 0) { - measurement.record(load); - } - } - } catch (Exception e) { - // com.sun.management may not be available - } - }); - } - - /** jvm.file_descriptor.count and jvm.file_descriptor.limit (UpDownCounter, Development). */ - private static void registerFileDescriptorMetrics(Meter meter) { - meter - .upDownCounterBuilder("jvm.file_descriptor.count") - .setDescription("Number of open file descriptors.") - .setUnit("{file_descriptor}") - .buildWithCallback( - measurement -> { - try { - java.lang.management.OperatingSystemMXBean osBean = - ManagementFactory.getOperatingSystemMXBean(); - if (osBean instanceof com.sun.management.UnixOperatingSystemMXBean) { - long count = - ((com.sun.management.UnixOperatingSystemMXBean) osBean) - .getOpenFileDescriptorCount(); - if (count >= 0) { - measurement.record(count); - } - } - } catch (Exception e) { - // UnixOperatingSystemMXBean not available on Windows - } - }); - - meter - .upDownCounterBuilder("jvm.file_descriptor.limit") - .setDescription("Maximum number of open file descriptors allowed.") - .setUnit("{file_descriptor}") - .buildWithCallback( - measurement -> { - try { - java.lang.management.OperatingSystemMXBean osBean = - ManagementFactory.getOperatingSystemMXBean(); - if (osBean instanceof com.sun.management.UnixOperatingSystemMXBean) { - long limit = - ((com.sun.management.UnixOperatingSystemMXBean) osBean) - .getMaxFileDescriptorCount(); - if (limit >= 0) { - measurement.record(limit); - } - } - } catch (Exception e) { - // UnixOperatingSystemMXBean not available on Windows - } - }); } private JvmOtlpRuntimeMetrics() {} diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy index cc31740700b..db689b8483f 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy @@ -37,6 +37,11 @@ class JvmOtlpRuntimeMetricsTest extends InstrumentationSpecification { then: def names = collector.metricNames.collect { it.toString() } + // Per OTel semconv, jvm.system.cpu.utilization, jvm.system.cpu.load_1m, + // jvm.file_descriptor.count and jvm.file_descriptor.limit are Opt-In and are + // intentionally not emitted yet — gating those behind a future + // DD_METRICS_OTEL_OPTIN_ENABLED flag is tracked separately. + // https://opentelemetry.io/docs/specs/semconv/general/metric-requirement-level/#opt-in def expectedMetrics = [ // Memory (5 metrics) "jvm.memory.used", @@ -54,21 +59,17 @@ class JvmOtlpRuntimeMetricsTest extends InstrumentationSpecification { "jvm.class.loaded", "jvm.class.count", "jvm.class.unloaded", - // CPU (4 metrics) + // CPU (3 metrics — jvm.system.cpu.utilization is Opt-In, omitted) "jvm.cpu.time", "jvm.cpu.count", "jvm.cpu.recent_utilization", - "jvm.system.cpu.utilization", - // File descriptors (2 metrics) - "jvm.file_descriptor.count", - "jvm.file_descriptor.limit", ] for (metric in expectedMetrics) { assert metric in names : "Expected metric '${metric}' not found. Got: ${names.sort()}" } - names.size() == 18 + names.size() == 15 // No DD-proprietary names should be present def ddNames = names.findAll { it.startsWith("jvm.heap_memory") || it.startsWith("jvm.thread_count") } From ffab1734441341a455f08286576a4b5a5cc22f57 Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Wed, 6 May 2026 15:41:10 -0400 Subject: [PATCH 07/15] Start OTLP runtime metrics from Agent boot, not OTel API advice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous wiring only fired when the application called OpenTelemetry.getMeterProvider() — which most JVM apps never do unless they themselves use the OTel API. Spring Boot, Akka, Vert.x and the system-tests weblogs don't, so JvmOtlpRuntimeMetrics.start() was never reached and zero jvm.* metrics were exported. Match the pattern Node and .NET use: start runtime metrics from the tracer's own init path (here, Agent.startJmx, the same place startJmxFetch lives). Reflective load through AGENT_CLASSLOADER mirrors startJmxFetch exactly. Gated on DD_RUNTIME_METRICS_ENABLED && OTel metrics enabled, both already set when the OTLP_RUNTIME_METRICS scenario runs. The OTLP exporter pipeline (OtlpMetricsService) is already started by CoreTracer when DD_METRICS_OTEL_ENABLED=true; this change only registers the JVM metric callbacks with OtelMeterProvider so the periodic export has something to collect. --- .../java/datadog/trace/bootstrap/Agent.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 5288f92dbe3..f21159a2728 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -26,6 +26,7 @@ import datadog.instrument.utils.ClassLoaderValue; import datadog.metrics.api.statsd.StatsDClientManager; import datadog.trace.api.Config; +import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.Platform; import datadog.trace.api.WithGlobalTracer; import datadog.trace.api.appsec.AppSecEventTracker; @@ -898,6 +899,13 @@ private static synchronized void startJmx() { if (jmxFetchEnabled) { startJmxFetch(); } + // OTLP runtime metrics start unconditionally when both DD_METRICS_OTEL_ENABLED + // and DD_RUNTIME_METRICS_ENABLED are set, regardless of whether the application + // imports the OTel API. The OTLP exporter (OtlpMetricsService) is already started + // by CoreTracer; this just registers the JVM metric callbacks with OtelMeterProvider. + if (Config.get().isRuntimeMetricsEnabled() && InstrumenterConfig.get().isMetricsOtelEnabled()) { + startOtlpRuntimeMetrics(); + } initializeJmxSystemAccessProvider(AGENT_CLASSLOADER); if (crashTrackingEnabled && CRASHTRACKER_INIT_AFTER_JMX != null) { try { @@ -989,6 +997,27 @@ private static synchronized void initializeJmxSystemAccessProvider( } } + /** + * Registers OTLP runtime metric callbacks (JVM heap, CPU, threads, classes, etc.) with the + * agent's OtelMeterProvider. The periodic OTLP exporter started by CoreTracer then collects and + * exports them — this is the same pattern Node and .NET use to start their runtime metrics + * unconditionally during tracer init, independent of any app-side OTel API usage. + */ + private static synchronized void startOtlpRuntimeMetrics() { + final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(AGENT_CLASSLOADER); + final Class jvmOtlpClass = + AGENT_CLASSLOADER.loadClass("datadog.opentelemetry.shim.metrics.JvmOtlpRuntimeMetrics"); + final Method startMethod = jvmOtlpClass.getMethod("start"); + startMethod.invoke(null); + } catch (final Throwable ex) { + log.error("Throwable thrown while starting OTLP runtime metrics", ex); + } finally { + safelySetContextClassLoader(contextLoader); + } + } + private static synchronized void startJmxFetch() { final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); try { From 728e3f1c680c0c6cd2e661aa088e07e8547e02a2 Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Wed, 6 May 2026 18:34:45 -0400 Subject: [PATCH 08/15] Drop Development-stability JVM metrics + migrate test to JUnit Two changes: 1. Strip 4 more JVM metrics that aren't Recommended per OTel semconv: jvm.memory.init, jvm.buffer.memory.used, jvm.buffer.memory.limit, jvm.buffer.count. They live in jvm-metrics-experimental.yaml with stability: development. Same partitioning OTel Java upstream applies in JmxRuntimeMetricsFactory's emitExperimentalTelemetry branch: non-experimental subset is exactly the 11 we now ship. Will return behind a future DD_METRICS_OTEL_OPTIN_ENABLED flag together with the previously-dropped Opt-In metrics (jvm.system.cpu.*, jvm.file_descriptor.*). Reference: https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/instrumentation/runtime-telemetry/library/src/main/java/io/opentelemetry/instrumentation/runtimetelemetry/internal/JmxRuntimeMetricsFactory.java 2. Migrate JvmOtlpRuntimeMetricsTest from Spock/Groovy to JUnit 5 Java per the new-Groovy-file enforcement workflow. Same 4 test cases, identical assertion semantics. Note: the opentelemetry-1.47 module still has 2 other pre-existing .groovy test files, so the module is not added to .github/g2j-migrated-modules.txt yet. --- .../shim/metrics/JvmOtlpRuntimeMetrics.java | 86 +------- .../metrics/JvmOtlpRuntimeMetricsTest.groovy | 167 --------------- .../metrics/JvmOtlpRuntimeMetricsTest.java | 197 ++++++++++++++++++ 3 files changed, 202 insertions(+), 248 deletions(-) delete mode 100644 dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy create mode 100644 dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java index 12bdb8007df..c845b435393 100644 --- a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java +++ b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java @@ -3,7 +3,6 @@ import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.Meter; -import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; import java.lang.management.MemoryPoolMXBean; @@ -43,7 +42,6 @@ public static void start() { try { Meter meter = OtelMeterProvider.INSTANCE.get(INSTRUMENTATION_SCOPE); registerMemoryMetrics(meter); - registerBufferMetrics(meter); registerThreadMetrics(meter); registerClassLoadingMetrics(meter); registerCpuMetrics(meter); @@ -58,8 +56,11 @@ public static void start() { // GC event durations needed to build a distribution. /** - * jvm.memory.used, jvm.memory.committed, jvm.memory.limit, jvm.memory.init, - * jvm.memory.used_after_last_gc — all UpDownCounter per spec. + * jvm.memory.used, jvm.memory.committed, jvm.memory.limit, jvm.memory.used_after_last_gc — all + * UpDownCounter per spec. + * + *

jvm.memory.init is Development per OTel semconv and is intentionally not emitted yet — + * gating it behind a future DD_METRICS_OTEL_OPTIN_ENABLED flag is tracked separately. */ private static void registerMemoryMetrics(Meter meter) { MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); @@ -145,26 +146,6 @@ private static void registerMemoryMetrics(Meter meter) { } }); - // jvm.memory.init (UpDownCounter, Development) - meter - .upDownCounterBuilder("jvm.memory.init") - .setDescription("Measure of initial memory requested.") - .setUnit("By") - .buildWithCallback( - measurement -> { - long heapInit = memoryBean.getHeapMemoryUsage().getInit(); - if (heapInit > 0) { - measurement.record( - heapInit, Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); - } - long nonHeapInit = memoryBean.getNonHeapMemoryUsage().getInit(); - if (nonHeapInit > 0) { - measurement.record( - nonHeapInit, - Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "non_heap")); - } - }); - // jvm.memory.used_after_last_gc (UpDownCounter, Stable) meter .upDownCounterBuilder("jvm.memory.used_after_last_gc") @@ -190,63 +171,6 @@ private static void registerMemoryMetrics(Meter meter) { }); } - /** jvm.buffer.* (UpDownCounter, Development) — JVM buffer pool metrics (direct, mapped). */ - private static void registerBufferMetrics(Meter meter) { - List bufferPools = - ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); - - meter - .upDownCounterBuilder("jvm.buffer.memory.used") - .setDescription("Measure of memory used by buffers.") - .setUnit("By") - .buildWithCallback( - measurement -> { - for (BufferPoolMXBean pool : bufferPools) { - long used = pool.getMemoryUsed(); - if (used >= 0) { - measurement.record( - used, - Attributes.of( - AttributeKey.stringKey("jvm.buffer.pool.name"), pool.getName())); - } - } - }); - - meter - .upDownCounterBuilder("jvm.buffer.memory.limit") - .setDescription("Measure of total memory capacity of buffers.") - .setUnit("By") - .buildWithCallback( - measurement -> { - for (BufferPoolMXBean pool : bufferPools) { - long limit = pool.getTotalCapacity(); - if (limit >= 0) { - measurement.record( - limit, - Attributes.of( - AttributeKey.stringKey("jvm.buffer.pool.name"), pool.getName())); - } - } - }); - - meter - .upDownCounterBuilder("jvm.buffer.count") - .setDescription("Number of buffers in the pool.") - .setUnit("{buffer}") - .buildWithCallback( - measurement -> { - for (BufferPoolMXBean pool : bufferPools) { - long count = pool.getCount(); - if (count >= 0) { - measurement.record( - count, - Attributes.of( - AttributeKey.stringKey("jvm.buffer.pool.name"), pool.getName())); - } - } - }); - } - /** jvm.thread.count (UpDownCounter, Stable) */ private static void registerThreadMetrics(Meter meter) { ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy deleted file mode 100644 index db689b8483f..00000000000 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/groovy/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.groovy +++ /dev/null @@ -1,167 +0,0 @@ -package opentelemetry147.metrics - -import datadog.opentelemetry.shim.metrics.JvmOtlpRuntimeMetrics -import datadog.opentelemetry.shim.metrics.OtelMeterProvider -import datadog.trace.agent.test.InstrumentationSpecification -import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope -import datadog.trace.bootstrap.otel.metrics.OtelInstrumentDescriptor -import datadog.trace.bootstrap.otel.metrics.data.OtelMetricRegistry -import datadog.trace.bootstrap.otlp.metrics.OtlpDataPoint -import datadog.trace.bootstrap.otlp.metrics.OtlpDoublePoint -import datadog.trace.bootstrap.otlp.metrics.OtlpLongPoint -import datadog.trace.bootstrap.otlp.metrics.OtlpMetricVisitor -import datadog.trace.bootstrap.otlp.metrics.OtlpMetricsVisitor -import datadog.trace.bootstrap.otlp.metrics.OtlpScopedMetricsVisitor - -/** - * Tests that JVM runtime metrics are registered and exported via OTLP - * using OTel semantic convention names (jvm.memory.used, jvm.thread.count, etc.). - * - * Ref: https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ - * Ref: https://github.com/DataDog/semantic-core/blob/main/sor/domains/metrics/integrations/java/_equivalence/ - */ -class JvmOtlpRuntimeMetricsTest extends InstrumentationSpecification { - - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig("dd.metrics.otel.enabled", "true") - } - - def "registers exactly 18 OTel-named JVM runtime metrics"() { - when: - JvmOtlpRuntimeMetrics.start() - def collector = new MetricCollector() - OtelMetricRegistry.INSTANCE.collectMetrics(collector) - - then: - def names = collector.metricNames.collect { it.toString() } - - // Per OTel semconv, jvm.system.cpu.utilization, jvm.system.cpu.load_1m, - // jvm.file_descriptor.count and jvm.file_descriptor.limit are Opt-In and are - // intentionally not emitted yet — gating those behind a future - // DD_METRICS_OTEL_OPTIN_ENABLED flag is tracked separately. - // https://opentelemetry.io/docs/specs/semconv/general/metric-requirement-level/#opt-in - def expectedMetrics = [ - // Memory (5 metrics) - "jvm.memory.used", - "jvm.memory.committed", - "jvm.memory.limit", - "jvm.memory.init", - "jvm.memory.used_after_last_gc", - // Buffers (3 metrics) - "jvm.buffer.memory.used", - "jvm.buffer.memory.limit", - "jvm.buffer.count", - // Threads (1 metric) - "jvm.thread.count", - // Classes (3 metrics) - "jvm.class.loaded", - "jvm.class.count", - "jvm.class.unloaded", - // CPU (3 metrics — jvm.system.cpu.utilization is Opt-In, omitted) - "jvm.cpu.time", - "jvm.cpu.count", - "jvm.cpu.recent_utilization", - ] - - for (metric in expectedMetrics) { - assert metric in names : "Expected metric '${metric}' not found. Got: ${names.sort()}" - } - - names.size() == 15 - - // No DD-proprietary names should be present - def ddNames = names.findAll { it.startsWith("jvm.heap_memory") || it.startsWith("jvm.thread_count") } - ddNames.isEmpty() - } - - def "jvm.memory.used has heap and non_heap type attributes"() { - when: - JvmOtlpRuntimeMetrics.start() - def collector = new MetricCollector() - OtelMetricRegistry.INSTANCE.collectMetrics(collector) - - then: - def types = collector.attributeValues("jvm.memory.used", "jvm.memory.type") - types.contains("heap") - types.contains("non_heap") - } - - def "jvm.memory.used heap value is positive"() { - when: - JvmOtlpRuntimeMetrics.start() - def collector = new MetricCollector() - OtelMetricRegistry.INSTANCE.collectMetrics(collector) - - then: - def heapPoints = collector.points["jvm.memory.used"] - .findAll { it.attrs["jvm.memory.type"] == "heap" } - heapPoints.size() > 0 - heapPoints[0].value > 0 - } - - def "jvm.thread.count is positive"() { - when: - JvmOtlpRuntimeMetrics.start() - def collector = new MetricCollector() - OtelMetricRegistry.INSTANCE.collectMetrics(collector) - - then: - def threadPoints = collector.points["jvm.thread.count"] - threadPoints.size() > 0 - threadPoints[0].value > 0 - } - - static class DataPointEntry { - Map attrs - Number value - } - - static class MetricCollector implements OtlpMetricsVisitor, OtlpScopedMetricsVisitor, OtlpMetricVisitor { - String currentInstrument = "" - Map currentAttrs = [:] - Set metricNames = new LinkedHashSet<>() - Map> points = [:].withDefault { [] } - - @Override - OtlpScopedMetricsVisitor visitScopedMetrics(OtelInstrumentationScope scope) { - return this - } - - @Override - OtlpMetricVisitor visitMetric(OtelInstrumentDescriptor descriptor) { - currentInstrument = descriptor.name.toString() - metricNames.add(descriptor.name.toString()) - return this - } - - @Override - void visitAttribute(int type, String key, Object value) { - currentAttrs.put(key.toString(), value.toString()) - } - - @Override - void visitDataPoint(OtlpDataPoint point) { - def attrs = new HashMap(currentAttrs) - currentAttrs.clear() - Number value = 0 - if (point instanceof OtlpLongPoint) { - value = ((OtlpLongPoint) point).value - } else if (point instanceof OtlpDoublePoint) { - value = ((OtlpDoublePoint) point).value - } - def entry = new DataPointEntry() - entry.attrs = attrs - entry.value = value - points[currentInstrument].add(entry) - } - - Set attributeValues(String metricName, String attrKey) { - points[metricName] - .collect { it.attrs[attrKey] } - .findAll { it != null } - .toSet() - } - } -} diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java new file mode 100644 index 00000000000..591026d2d8f --- /dev/null +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java @@ -0,0 +1,197 @@ +package opentelemetry147.metrics; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.opentelemetry.shim.metrics.JvmOtlpRuntimeMetrics; +import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; +import datadog.trace.bootstrap.otel.metrics.OtelInstrumentDescriptor; +import datadog.trace.bootstrap.otel.metrics.data.OtelMetricRegistry; +import datadog.trace.bootstrap.otlp.metrics.OtlpDataPoint; +import datadog.trace.bootstrap.otlp.metrics.OtlpDoublePoint; +import datadog.trace.bootstrap.otlp.metrics.OtlpLongPoint; +import datadog.trace.bootstrap.otlp.metrics.OtlpMetricVisitor; +import datadog.trace.bootstrap.otlp.metrics.OtlpMetricsVisitor; +import datadog.trace.bootstrap.otlp.metrics.OtlpScopedMetricsVisitor; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Tests that JVM runtime metrics are registered and exported via OTLP using OTel semantic + * convention names (jvm.memory.used, jvm.thread.count, etc.). + * + *

Ref: https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ + * + *

Ref: + * https://github.com/DataDog/semantic-core/blob/main/sor/domains/metrics/integrations/java/_equivalence/ + */ +public class JvmOtlpRuntimeMetricsTest { + + @BeforeAll + static void setUp() { + System.setProperty("dd.metrics.otel.enabled", "true"); + JvmOtlpRuntimeMetrics.start(); + } + + @Test + void registersExactly11RecommendedJvmMetrics() { + MetricCollector collector = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(collector); + + // Per OTel semconv, the following are Opt-In or Development-stability and are intentionally + // not emitted yet — gating them behind a future DD_METRICS_OTEL_OPTIN_ENABLED flag is tracked + // separately. Same partitioning OTel Java upstream uses (see JmxRuntimeMetricsFactory's + // emitExperimentalTelemetry branch): + // Opt-In: jvm.system.cpu.utilization, jvm.system.cpu.load_1m, + // jvm.file_descriptor.count, jvm.file_descriptor.limit + // Development: jvm.memory.init, jvm.buffer.memory.used, jvm.buffer.memory.limit, + // jvm.buffer.count + // https://opentelemetry.io/docs/specs/semconv/general/metric-requirement-level/#opt-in + List expectedMetrics = + Arrays.asList( + // Memory (4 metrics) + "jvm.memory.used", + "jvm.memory.committed", + "jvm.memory.limit", + "jvm.memory.used_after_last_gc", + // Threads (1 metric) + "jvm.thread.count", + // Classes (3 metrics) + "jvm.class.loaded", + "jvm.class.count", + "jvm.class.unloaded", + // CPU (3 metrics) + "jvm.cpu.time", + "jvm.cpu.count", + "jvm.cpu.recent_utilization"); + + Set names = collector.metricNames; + for (String metric : expectedMetrics) { + assertTrue( + names.contains(metric), + "Expected metric '" + metric + "' not found. Got: " + new java.util.TreeSet<>(names)); + } + + assertEquals(11, names.size(), "Expected 11 metrics, got: " + new java.util.TreeSet<>(names)); + + // No DD-proprietary names should be present + List ddNames = + names.stream() + .filter(n -> n.startsWith("jvm.heap_memory") || n.startsWith("jvm.thread_count")) + .collect(Collectors.toList()); + assertTrue(ddNames.isEmpty(), "DD-proprietary names leaked: " + ddNames); + } + + @Test + void jvmMemoryUsedHasHeapAndNonHeapTypeAttributes() { + MetricCollector collector = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(collector); + + Set types = collector.attributeValues("jvm.memory.used", "jvm.memory.type"); + assertTrue(types.contains("heap"), "jvm.memory.used should have heap attribute"); + assertTrue(types.contains("non_heap"), "jvm.memory.used should have non_heap attribute"); + } + + @Test + void jvmMemoryUsedHeapValueIsPositive() { + MetricCollector collector = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(collector); + + List points = collector.points.get("jvm.memory.used"); + assertNotNull(points, "jvm.memory.used should have data points"); + List heapPoints = + points.stream() + .filter(p -> "heap".equals(p.attrs.get("jvm.memory.type"))) + .collect(Collectors.toList()); + assertTrue(!heapPoints.isEmpty(), "jvm.memory.used should have heap data point"); + assertTrue( + heapPoints.get(0).value.longValue() > 0, + "jvm.memory.used heap value should be positive, got " + heapPoints.get(0).value); + } + + @Test + void jvmThreadCountIsPositive() { + MetricCollector collector = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(collector); + + List threadPoints = collector.points.get("jvm.thread.count"); + assertNotNull(threadPoints, "jvm.thread.count should have data points"); + assertTrue(!threadPoints.isEmpty(), "jvm.thread.count should have data points"); + assertTrue( + threadPoints.get(0).value.longValue() > 0, + "jvm.thread.count value should be positive, got " + threadPoints.get(0).value); + } + + static final class DataPointEntry { + final Map attrs; + final Number value; + + DataPointEntry(Map attrs, Number value) { + this.attrs = attrs; + this.value = value; + } + } + + static final class MetricCollector + implements OtlpMetricsVisitor, OtlpScopedMetricsVisitor, OtlpMetricVisitor { + + String currentInstrument = ""; + final Map currentAttrs = new LinkedHashMap<>(); + final Set metricNames = new LinkedHashSet<>(); + final Map> points = new LinkedHashMap<>(); + + @Override + public OtlpScopedMetricsVisitor visitScopedMetrics(OtelInstrumentationScope scope) { + return this; + } + + @Override + public OtlpMetricVisitor visitMetric(OtelInstrumentDescriptor descriptor) { + currentInstrument = descriptor.getName().toString(); + metricNames.add(currentInstrument); + points.computeIfAbsent(currentInstrument, k -> new java.util.ArrayList<>()); + return this; + } + + @Override + public void visitAttribute(int type, String key, Object value) { + currentAttrs.put(key, value == null ? null : value.toString()); + } + + @Override + public void visitDataPoint(OtlpDataPoint point) { + Map attrs = new HashMap<>(currentAttrs); + currentAttrs.clear(); + Number value = 0; + if (point instanceof OtlpLongPoint) { + value = ((OtlpLongPoint) point).value; + } else if (point instanceof OtlpDoublePoint) { + value = ((OtlpDoublePoint) point).value; + } + points + .computeIfAbsent(currentInstrument, k -> new java.util.ArrayList<>()) + .add(new DataPointEntry(attrs, value)); + } + + Set attributeValues(String metricName, String attrKey) { + List entries = points.get(metricName); + if (entries == null) { + return new LinkedHashSet<>(); + } + return entries.stream() + .map(e -> e.attrs.get(attrKey)) + .filter(java.util.Objects::nonNull) + .map(Object::toString) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + } +} From 67d91a9d16a15ca02e884f59d32d6b6ae9c9d446 Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Wed, 6 May 2026 18:56:08 -0400 Subject: [PATCH 09/15] Move OTLP runtime metrics start out of the JMX 15s delay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Calling startOtlpRuntimeMetrics() from startJmx() inherits its 15-second jitter delay, which races with short-lived test scenarios — for slower weblogs (ratpack hit this), OtelMetricRegistry callbacks are registered after the OtlpMetricsService has already done its first scheduled flush, so the agent receives 0 jvm.* metrics for the test's wait window. The runtime metric callbacks don't actually need JMX initialized — they read ManagementFactory directly. Move the call into installDatadogTracer right after CoreTracer (and OtlpMetricsService) are up. Same effective gating (DD_RUNTIME_METRICS_ENABLED + DD_METRICS_OTEL_ENABLED), no delay. --- .../java/datadog/trace/bootstrap/Agent.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index f21159a2728..d5c2290055b 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -847,6 +847,17 @@ private static synchronized void installDatadogTracer( initTelemetry.onFatalError(ex); } + // OTLP runtime metrics start unconditionally when both DD_METRICS_OTEL_ENABLED + // and DD_RUNTIME_METRICS_ENABLED are set, regardless of whether the application + // imports the OTel API. The OTLP exporter (OtlpMetricsService) was just started + // by CoreTracer above; this registers the JVM metric callbacks with + // OtelMeterProvider so the periodic export has data to collect. Done here + // (not in startJmx, which is delayed 15s) so callbacks are registered before + // the exporter's first scheduled flush — otherwise short-lived tests miss data. + if (Config.get().isRuntimeMetricsEnabled() && InstrumenterConfig.get().isMetricsOtelEnabled()) { + startOtlpRuntimeMetrics(); + } + StaticEventLogger.end("GlobalTracer"); } @@ -899,13 +910,6 @@ private static synchronized void startJmx() { if (jmxFetchEnabled) { startJmxFetch(); } - // OTLP runtime metrics start unconditionally when both DD_METRICS_OTEL_ENABLED - // and DD_RUNTIME_METRICS_ENABLED are set, regardless of whether the application - // imports the OTel API. The OTLP exporter (OtlpMetricsService) is already started - // by CoreTracer; this just registers the JVM metric callbacks with OtelMeterProvider. - if (Config.get().isRuntimeMetricsEnabled() && InstrumenterConfig.get().isMetricsOtelEnabled()) { - startOtlpRuntimeMetrics(); - } initializeJmxSystemAccessProvider(AGENT_CLASSLOADER); if (crashTrackingEnabled && CRASHTRACKER_INIT_AFTER_JMX != null) { try { From fbbd69c625df318ba9c7f8ac1d3bbd47292b1811 Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Thu, 7 May 2026 07:37:45 -0400 Subject: [PATCH 10/15] Remove redundant JvmOtlpRuntimeMetrics.start() from MeterProviderAdvice Agent.installDatadogTracer now triggers runtime metric registration on agent boot (the canonical entry point), so the duplicate call from the OpenTelemetry.getMeterProvider advice is dead code. JvmOtlpRuntimeMetrics is already idempotent so removing the second invocation is a no-op behaviorally; this just reduces review surface. --- .../OpenTelemetryMetricsInstrumentation.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/main/java/datadog/trace/instrumentation/opentelemetry147/OpenTelemetryMetricsInstrumentation.java b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/main/java/datadog/trace/instrumentation/opentelemetry147/OpenTelemetryMetricsInstrumentation.java index b293687ad0f..4cb6fd79ede 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/main/java/datadog/trace/instrumentation/opentelemetry147/OpenTelemetryMetricsInstrumentation.java +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/main/java/datadog/trace/instrumentation/opentelemetry147/OpenTelemetryMetricsInstrumentation.java @@ -7,7 +7,6 @@ import static net.bytebuddy.matcher.ElementMatchers.takesNoArguments; import com.google.auto.service.AutoService; -import datadog.opentelemetry.shim.metrics.JvmOtlpRuntimeMetrics; import datadog.opentelemetry.shim.metrics.OtelMeterProvider; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; @@ -102,12 +101,6 @@ public static class MeterProviderAdvice { @Advice.OnMethodExit(suppress = Throwable.class) public static void returnProvider(@Advice.Return(readOnly = false) MeterProvider result) { result = OtelMeterProvider.INSTANCE; - // Start JVM runtime metrics when both DD_METRICS_OTEL_ENABLED and - // DD_RUNTIME_METRICS_ENABLED are true, matching the .NET/Go/NodeJS pattern. - // JvmOtlpRuntimeMetrics.start() is idempotent (checks a started flag internally). - if (datadog.trace.api.Config.get().isRuntimeMetricsEnabled()) { - JvmOtlpRuntimeMetrics.start(); - } } public static void muzzleCheck(DoubleGauge doubleGauge) { From 238d62ee9677b3f4c86c648261334623c6b0b404 Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Thu, 7 May 2026 07:48:04 -0400 Subject: [PATCH 11/15] Register JvmOtlpRuntimeMetrics for native-image reflection Mirrors OTel Java upstream's runtime-telemetry reflect-config.json: GraalVM AOT analysis can't see Class.forName(...) lookups through AGENT_CLASSLOADER, so the class gets dead-stripped from the native binary unless explicitly listed. Without this entry, the Agent.startOtlpRuntimeMetrics() reflection in installDatadogTracer fails on native-image with ClassNotFoundException and zero jvm.* metrics are emitted. --- .../com.datadoghq/dd-java-agent/reflect-config.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json b/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json index 3225d6dd59f..a3470425d35 100644 --- a/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json +++ b/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json @@ -218,5 +218,11 @@ "methods": [ {"name": "", "parameterTypes": []} ] + }, + { + "name": "datadog.trace.bootstrap.otel.shim.metrics.JvmOtlpRuntimeMetrics", + "methods": [ + {"name": "start", "parameterTypes": []} + ] } ] From 37635d4cfc31da4c9a530599410107ca24b88fec Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Thu, 7 May 2026 12:04:15 -0400 Subject: [PATCH 12/15] Re-add Development-stability JVM metrics (non-opt-in) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ship the 4 metrics that are stability:development per OTel semconv but NOT marked Opt-In: jvm.memory.init, jvm.buffer.memory.used, jvm.buffer.memory.limit, jvm.buffer.count. Final set is 15 metrics — everything OTel ships except the 4 explicit Opt-In ones (jvm.system.cpu.*, jvm.file_descriptor.*). --- .../shim/metrics/JvmOtlpRuntimeMetrics.java | 87 +++++++++++++++++-- .../metrics/JvmOtlpRuntimeMetricsTest.java | 21 ++--- 2 files changed, 88 insertions(+), 20 deletions(-) diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java index c845b435393..0e50f434789 100644 --- a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java +++ b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java @@ -3,6 +3,7 @@ import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.Meter; +import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; import java.lang.management.MemoryPoolMXBean; @@ -42,6 +43,7 @@ public static void start() { try { Meter meter = OtelMeterProvider.INSTANCE.get(INSTRUMENTATION_SCOPE); registerMemoryMetrics(meter); + registerBufferMetrics(meter); registerThreadMetrics(meter); registerClassLoadingMetrics(meter); registerCpuMetrics(meter); @@ -56,11 +58,8 @@ public static void start() { // GC event durations needed to build a distribution. /** - * jvm.memory.used, jvm.memory.committed, jvm.memory.limit, jvm.memory.used_after_last_gc — all - * UpDownCounter per spec. - * - *

jvm.memory.init is Development per OTel semconv and is intentionally not emitted yet — - * gating it behind a future DD_METRICS_OTEL_OPTIN_ENABLED flag is tracked separately. + * jvm.memory.used, jvm.memory.committed, jvm.memory.limit, jvm.memory.init, + * jvm.memory.used_after_last_gc — all UpDownCounter per spec. */ private static void registerMemoryMetrics(Meter meter) { MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); @@ -146,6 +145,26 @@ private static void registerMemoryMetrics(Meter meter) { } }); + // jvm.memory.init (UpDownCounter, Development) + meter + .upDownCounterBuilder("jvm.memory.init") + .setDescription("Measure of initial memory requested.") + .setUnit("By") + .buildWithCallback( + measurement -> { + long heapInit = memoryBean.getHeapMemoryUsage().getInit(); + if (heapInit > 0) { + measurement.record( + heapInit, Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); + } + long nonHeapInit = memoryBean.getNonHeapMemoryUsage().getInit(); + if (nonHeapInit > 0) { + measurement.record( + nonHeapInit, + Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "non_heap")); + } + }); + // jvm.memory.used_after_last_gc (UpDownCounter, Stable) meter .upDownCounterBuilder("jvm.memory.used_after_last_gc") @@ -171,6 +190,64 @@ private static void registerMemoryMetrics(Meter meter) { }); } + /** jvm.thread.count (UpDownCounter, Stable) */ + /** jvm.buffer.* (UpDownCounter, Development) — JVM buffer pool metrics (direct, mapped). */ + private static void registerBufferMetrics(Meter meter) { + List bufferPools = + ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); + + meter + .upDownCounterBuilder("jvm.buffer.memory.used") + .setDescription("Measure of memory used by buffers.") + .setUnit("By") + .buildWithCallback( + measurement -> { + for (BufferPoolMXBean pool : bufferPools) { + long used = pool.getMemoryUsed(); + if (used >= 0) { + measurement.record( + used, + Attributes.of( + AttributeKey.stringKey("jvm.buffer.pool.name"), pool.getName())); + } + } + }); + + meter + .upDownCounterBuilder("jvm.buffer.memory.limit") + .setDescription("Measure of total memory capacity of buffers.") + .setUnit("By") + .buildWithCallback( + measurement -> { + for (BufferPoolMXBean pool : bufferPools) { + long limit = pool.getTotalCapacity(); + if (limit >= 0) { + measurement.record( + limit, + Attributes.of( + AttributeKey.stringKey("jvm.buffer.pool.name"), pool.getName())); + } + } + }); + + meter + .upDownCounterBuilder("jvm.buffer.count") + .setDescription("Number of buffers in the pool.") + .setUnit("{buffer}") + .buildWithCallback( + measurement -> { + for (BufferPoolMXBean pool : bufferPools) { + long count = pool.getCount(); + if (count >= 0) { + measurement.record( + count, + Attributes.of( + AttributeKey.stringKey("jvm.buffer.pool.name"), pool.getName())); + } + } + }); + } + /** jvm.thread.count (UpDownCounter, Stable) */ private static void registerThreadMetrics(Meter meter) { ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java index 591026d2d8f..f447a44601c 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java @@ -43,33 +43,24 @@ static void setUp() { } @Test - void registersExactly11RecommendedJvmMetrics() { + void registersExpectedJvmMetrics() { MetricCollector collector = new MetricCollector(); OtelMetricRegistry.INSTANCE.collectMetrics(collector); - // Per OTel semconv, the following are Opt-In or Development-stability and are intentionally - // not emitted yet — gating them behind a future DD_METRICS_OTEL_OPTIN_ENABLED flag is tracked - // separately. Same partitioning OTel Java upstream uses (see JmxRuntimeMetricsFactory's - // emitExperimentalTelemetry branch): - // Opt-In: jvm.system.cpu.utilization, jvm.system.cpu.load_1m, - // jvm.file_descriptor.count, jvm.file_descriptor.limit - // Development: jvm.memory.init, jvm.buffer.memory.used, jvm.buffer.memory.limit, - // jvm.buffer.count - // https://opentelemetry.io/docs/specs/semconv/general/metric-requirement-level/#opt-in List expectedMetrics = Arrays.asList( - // Memory (4 metrics) "jvm.memory.used", "jvm.memory.committed", "jvm.memory.limit", + "jvm.memory.init", "jvm.memory.used_after_last_gc", - // Threads (1 metric) + "jvm.buffer.memory.used", + "jvm.buffer.memory.limit", + "jvm.buffer.count", "jvm.thread.count", - // Classes (3 metrics) "jvm.class.loaded", "jvm.class.count", "jvm.class.unloaded", - // CPU (3 metrics) "jvm.cpu.time", "jvm.cpu.count", "jvm.cpu.recent_utilization"); @@ -81,7 +72,7 @@ void registersExactly11RecommendedJvmMetrics() { "Expected metric '" + metric + "' not found. Got: " + new java.util.TreeSet<>(names)); } - assertEquals(11, names.size(), "Expected 11 metrics, got: " + new java.util.TreeSet<>(names)); + assertEquals(15, names.size(), "Expected 15 metrics, got: " + new java.util.TreeSet<>(names)); // No DD-proprietary names should be present List ddNames = From 6d1b9343a75f149cc3a51f1b2d6985612336f989 Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Thu, 7 May 2026 17:24:44 -0400 Subject: [PATCH 13/15] Tighten JvmOtlpRuntimeMetrics: trim comments, dedupe via helpers - Strip section comments that duplicate the next line (// jvm.memory.used etc.) - Remove DD_METRICS_OTEL_OPTIN_ENABLED future-flag note from the CPU JavaDoc - Tighten file header and the jvm.gc.duration exclusion note - Fix stray duplicate JavaDoc that was floating above registerBufferMetrics - Lift jvm.memory.type / jvm.memory.pool.name / jvm.buffer.pool.name to static AttributeKey constants - Extract bufferPoolMetric(..., ToLongFunction) helper used by all 3 jvm.buffer.* metrics (~50 lines deduplicated) - Extract poolAttributes(MemoryPoolMXBean) helper for the per-pool memory tag set (8 callsites) - Extract sunOsBean() helper so the com.sun.management cast lives in one place (was duplicated by jvm.cpu.time and jvm.cpu.recent_utilization) No metric names, types, units, or attributes change. JvmOtlpRuntimeMetricsTest still asserts the same 15 metrics with positive values + correct attributes. --- .../shim/metrics/JvmOtlpRuntimeMetrics.java | 262 +++++++----------- 1 file changed, 102 insertions(+), 160 deletions(-) diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java index 0e50f434789..1913f206ebb 100644 --- a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java +++ b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java @@ -1,5 +1,6 @@ package datadog.opentelemetry.shim.metrics; +import com.sun.management.OperatingSystemMXBean; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.Meter; @@ -10,26 +11,23 @@ import java.lang.management.MemoryUsage; import java.lang.management.ThreadMXBean; import java.util.List; +import java.util.function.ToLongFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Registers JVM runtime metrics using OTel semantic convention names via the dd-trace-java OTLP - * metrics pipeline. These metrics flow via OTLP without requiring a Datadog Agent or DogStatsD. - * - *

Only includes metrics where we can match the exact OTel spec type. Metrics requiring Histogram - * type (jvm.gc.duration) are excluded because JMX cannot produce distribution data. - * - *

OTel JVM runtime metrics conventions: - * https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ - * - *

Semantic-core equivalence mappings: - * https://github.com/DataDog/semantic-core/blob/main/sor/domains/metrics/integrations/java/_equivalence/ + * Registers JVM runtime metrics with OTel-native names against the agent's MeterProvider. See + * https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/. */ public final class JvmOtlpRuntimeMetrics { private static final Logger log = LoggerFactory.getLogger(JvmOtlpRuntimeMetrics.class); private static final String INSTRUMENTATION_SCOPE = "datadog.jvm.runtime"; + private static final AttributeKey MEMORY_TYPE = AttributeKey.stringKey("jvm.memory.type"); + private static final AttributeKey MEMORY_POOL = + AttributeKey.stringKey("jvm.memory.pool.name"); + private static final AttributeKey BUFFER_POOL = + AttributeKey.stringKey("jvm.buffer.pool.name"); private static volatile boolean started = false; @@ -53,9 +51,7 @@ public static void start() { } } - // Note: jvm.gc.duration is excluded — OTel spec requires Histogram type but JMX only provides - // cumulative milliseconds via GarbageCollectorMXBean.getCollectionTime(), not individual - // GC event durations needed to build a distribution. + // jvm.gc.duration is excluded — spec requires Histogram, JMX only exposes cumulative time. /** * jvm.memory.used, jvm.memory.committed, jvm.memory.limit, jvm.memory.init, @@ -65,7 +61,6 @@ private static void registerMemoryMetrics(Meter meter) { MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); List pools = ManagementFactory.getMemoryPoolMXBeans(); - // jvm.memory.used (UpDownCounter, Stable) meter .upDownCounterBuilder("jvm.memory.used") .setDescription("Measure of memory used.") @@ -73,23 +68,15 @@ private static void registerMemoryMetrics(Meter meter) { .buildWithCallback( measurement -> { measurement.record( - memoryBean.getHeapMemoryUsage().getUsed(), - Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); + memoryBean.getHeapMemoryUsage().getUsed(), Attributes.of(MEMORY_TYPE, "heap")); measurement.record( memoryBean.getNonHeapMemoryUsage().getUsed(), - Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "non_heap")); + Attributes.of(MEMORY_TYPE, "non_heap")); for (MemoryPoolMXBean pool : pools) { - measurement.record( - pool.getUsage().getUsed(), - Attributes.of( - AttributeKey.stringKey("jvm.memory.type"), - pool.getType().name().toLowerCase(), - AttributeKey.stringKey("jvm.memory.pool.name"), - pool.getName())); + measurement.record(pool.getUsage().getUsed(), poolAttributes(pool)); } }); - // jvm.memory.committed (UpDownCounter, Stable) meter .upDownCounterBuilder("jvm.memory.committed") .setDescription("Measure of memory committed.") @@ -98,22 +85,15 @@ private static void registerMemoryMetrics(Meter meter) { measurement -> { measurement.record( memoryBean.getHeapMemoryUsage().getCommitted(), - Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); + Attributes.of(MEMORY_TYPE, "heap")); measurement.record( memoryBean.getNonHeapMemoryUsage().getCommitted(), - Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "non_heap")); + Attributes.of(MEMORY_TYPE, "non_heap")); for (MemoryPoolMXBean pool : pools) { - measurement.record( - pool.getUsage().getCommitted(), - Attributes.of( - AttributeKey.stringKey("jvm.memory.type"), - pool.getType().name().toLowerCase(), - AttributeKey.stringKey("jvm.memory.pool.name"), - pool.getName())); + measurement.record(pool.getUsage().getCommitted(), poolAttributes(pool)); } }); - // jvm.memory.limit (UpDownCounter, Stable) meter .upDownCounterBuilder("jvm.memory.limit") .setDescription("Measure of max obtainable memory.") @@ -122,30 +102,20 @@ private static void registerMemoryMetrics(Meter meter) { measurement -> { long heapMax = memoryBean.getHeapMemoryUsage().getMax(); if (heapMax > 0) { - measurement.record( - heapMax, Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); + measurement.record(heapMax, Attributes.of(MEMORY_TYPE, "heap")); } long nonHeapMax = memoryBean.getNonHeapMemoryUsage().getMax(); if (nonHeapMax > 0) { - measurement.record( - nonHeapMax, - Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "non_heap")); + measurement.record(nonHeapMax, Attributes.of(MEMORY_TYPE, "non_heap")); } for (MemoryPoolMXBean pool : pools) { long max = pool.getUsage().getMax(); if (max > 0) { - measurement.record( - max, - Attributes.of( - AttributeKey.stringKey("jvm.memory.type"), - pool.getType().name().toLowerCase(), - AttributeKey.stringKey("jvm.memory.pool.name"), - pool.getName())); + measurement.record(max, poolAttributes(pool)); } } }); - // jvm.memory.init (UpDownCounter, Development) meter .upDownCounterBuilder("jvm.memory.init") .setDescription("Measure of initial memory requested.") @@ -154,18 +124,14 @@ private static void registerMemoryMetrics(Meter meter) { measurement -> { long heapInit = memoryBean.getHeapMemoryUsage().getInit(); if (heapInit > 0) { - measurement.record( - heapInit, Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "heap")); + measurement.record(heapInit, Attributes.of(MEMORY_TYPE, "heap")); } long nonHeapInit = memoryBean.getNonHeapMemoryUsage().getInit(); if (nonHeapInit > 0) { - measurement.record( - nonHeapInit, - Attributes.of(AttributeKey.stringKey("jvm.memory.type"), "non_heap")); + measurement.record(nonHeapInit, Attributes.of(MEMORY_TYPE, "non_heap")); } }); - // jvm.memory.used_after_last_gc (UpDownCounter, Stable) meter .upDownCounterBuilder("jvm.memory.used_after_last_gc") .setDescription("Measure of memory used after the most recent garbage collection event.") @@ -174,84 +140,43 @@ private static void registerMemoryMetrics(Meter meter) { measurement -> { for (MemoryPoolMXBean pool : pools) { MemoryUsage collectionUsage = pool.getCollectionUsage(); - if (collectionUsage != null) { - long used = collectionUsage.getUsed(); - if (used >= 0) { - measurement.record( - used, - Attributes.of( - AttributeKey.stringKey("jvm.memory.type"), - pool.getType().name().toLowerCase(), - AttributeKey.stringKey("jvm.memory.pool.name"), - pool.getName())); - } + if (collectionUsage != null && collectionUsage.getUsed() >= 0) { + measurement.record(collectionUsage.getUsed(), poolAttributes(pool)); } } }); } - /** jvm.thread.count (UpDownCounter, Stable) */ - /** jvm.buffer.* (UpDownCounter, Development) — JVM buffer pool metrics (direct, mapped). */ + /** jvm.buffer.* (UpDownCounter, Development) — direct + mapped pool metrics. */ private static void registerBufferMetrics(Meter meter) { List bufferPools = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); - - meter - .upDownCounterBuilder("jvm.buffer.memory.used") - .setDescription("Measure of memory used by buffers.") - .setUnit("By") - .buildWithCallback( - measurement -> { - for (BufferPoolMXBean pool : bufferPools) { - long used = pool.getMemoryUsed(); - if (used >= 0) { - measurement.record( - used, - Attributes.of( - AttributeKey.stringKey("jvm.buffer.pool.name"), pool.getName())); - } - } - }); - - meter - .upDownCounterBuilder("jvm.buffer.memory.limit") - .setDescription("Measure of total memory capacity of buffers.") - .setUnit("By") - .buildWithCallback( - measurement -> { - for (BufferPoolMXBean pool : bufferPools) { - long limit = pool.getTotalCapacity(); - if (limit >= 0) { - measurement.record( - limit, - Attributes.of( - AttributeKey.stringKey("jvm.buffer.pool.name"), pool.getName())); - } - } - }); - - meter - .upDownCounterBuilder("jvm.buffer.count") - .setDescription("Number of buffers in the pool.") - .setUnit("{buffer}") - .buildWithCallback( - measurement -> { - for (BufferPoolMXBean pool : bufferPools) { - long count = pool.getCount(); - if (count >= 0) { - measurement.record( - count, - Attributes.of( - AttributeKey.stringKey("jvm.buffer.pool.name"), pool.getName())); - } - } - }); + bufferPoolMetric( + meter, + "jvm.buffer.memory.used", + "Measure of memory used by buffers.", + "By", + bufferPools, + BufferPoolMXBean::getMemoryUsed); + bufferPoolMetric( + meter, + "jvm.buffer.memory.limit", + "Measure of total memory capacity of buffers.", + "By", + bufferPools, + BufferPoolMXBean::getTotalCapacity); + bufferPoolMetric( + meter, + "jvm.buffer.count", + "Number of buffers in the pool.", + "{buffer}", + bufferPools, + BufferPoolMXBean::getCount); } - /** jvm.thread.count (UpDownCounter, Stable) */ + /** jvm.thread.count (UpDownCounter, Stable). */ private static void registerThreadMetrics(Meter meter) { ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); - meter .upDownCounterBuilder("jvm.thread.count") .setDescription("Number of executing platform threads.") @@ -260,12 +185,10 @@ private static void registerThreadMetrics(Meter meter) { } /** - * jvm.class.loaded (Counter, Stable) — cumulative total loaded since JVM start. - * jvm.class.unloaded (Counter, Stable) — cumulative total unloaded since JVM start. - * jvm.class.count (UpDownCounter, Stable) — currently loaded count. + * jvm.class.loaded (Counter), jvm.class.unloaded (Counter), jvm.class.count (UpDownCounter) — all + * Stable per spec. */ private static void registerClassLoadingMetrics(Meter meter) { - // jvm.class.loaded — Counter per spec (cumulative total, only goes up) meter .counterBuilder("jvm.class.loaded") .setDescription("Number of classes loaded since JVM start.") @@ -275,7 +198,6 @@ private static void registerClassLoadingMetrics(Meter meter) { measurement.record( ManagementFactory.getClassLoadingMXBean().getTotalLoadedClassCount())); - // jvm.class.count — UpDownCounter per spec (current count, can decrease) meter .upDownCounterBuilder("jvm.class.count") .setDescription("Number of classes currently loaded.") @@ -285,7 +207,6 @@ private static void registerClassLoadingMetrics(Meter meter) { measurement.record( ManagementFactory.getClassLoadingMXBean().getLoadedClassCount())); - // jvm.class.unloaded — Counter per spec meter .counterBuilder("jvm.class.unloaded") .setDescription("Number of classes unloaded since JVM start.") @@ -297,16 +218,10 @@ private static void registerClassLoadingMetrics(Meter meter) { } /** - * jvm.cpu.time (Counter, Stable), jvm.cpu.count (UpDownCounter, Stable), - * jvm.cpu.recent_utilization (Gauge, Stable). - * - *

jvm.system.cpu.utilization, jvm.system.cpu.load_1m are Opt-In per OTel semconv and are - * intentionally not emitted; gating those behind a future DD_METRICS_OTEL_OPTIN_ENABLED flag is - * tracked separately. See - * https://opentelemetry.io/docs/specs/semconv/general/metric-requirement-level/#opt-in. + * jvm.cpu.time (Counter), jvm.cpu.count (UpDownCounter), jvm.cpu.recent_utilization (Gauge) — all + * Stable per spec. */ private static void registerCpuMetrics(Meter meter) { - // jvm.cpu.time — Counter per spec (cumulative CPU time in seconds) meter .counterBuilder("jvm.cpu.time") .ofDoubles() @@ -314,22 +229,16 @@ private static void registerCpuMetrics(Meter meter) { .setUnit("s") .buildWithCallback( measurement -> { - try { - java.lang.management.OperatingSystemMXBean osBean = - ManagementFactory.getOperatingSystemMXBean(); - if (osBean instanceof com.sun.management.OperatingSystemMXBean) { - long nanos = - ((com.sun.management.OperatingSystemMXBean) osBean).getProcessCpuTime(); - if (nanos >= 0) { - measurement.record(nanos / 1e9); - } - } - } catch (Exception e) { - // com.sun.management may not be available + OperatingSystemMXBean osBean = sunOsBean(); + if (osBean == null) { + return; + } + long nanos = osBean.getProcessCpuTime(); + if (nanos >= 0) { + measurement.record(nanos / 1e9); } }); - // jvm.cpu.count — UpDownCounter per spec meter .upDownCounterBuilder("jvm.cpu.count") .setDescription("Number of processors available to the JVM.") @@ -337,28 +246,61 @@ private static void registerCpuMetrics(Meter meter) { .buildWithCallback( measurement -> measurement.record(Runtime.getRuntime().availableProcessors())); - // jvm.cpu.recent_utilization — Gauge per spec meter .gaugeBuilder("jvm.cpu.recent_utilization") .setDescription("Recent CPU utilization for the process as reported by the JVM.") .setUnit("1") .buildWithCallback( measurement -> { - try { - java.lang.management.OperatingSystemMXBean osBean = - ManagementFactory.getOperatingSystemMXBean(); - if (osBean instanceof com.sun.management.OperatingSystemMXBean) { - double cpuLoad = - ((com.sun.management.OperatingSystemMXBean) osBean).getProcessCpuLoad(); - if (cpuLoad >= 0) { - measurement.record(cpuLoad); - } + OperatingSystemMXBean osBean = sunOsBean(); + if (osBean == null) { + return; + } + double cpuLoad = osBean.getProcessCpuLoad(); + if (cpuLoad >= 0) { + measurement.record(cpuLoad); + } + }); + } + + /** + * Builds an UpDownCounter that iterates each platform buffer pool and records {@code getter} with + * the {@code jvm.buffer.pool.name} attribute. Skips negative readings. + */ + private static void bufferPoolMetric( + Meter meter, + String name, + String description, + String unit, + List bufferPools, + ToLongFunction getter) { + meter + .upDownCounterBuilder(name) + .setDescription(description) + .setUnit(unit) + .buildWithCallback( + measurement -> { + for (BufferPoolMXBean pool : bufferPools) { + long value = getter.applyAsLong(pool); + if (value >= 0) { + measurement.record(value, Attributes.of(BUFFER_POOL, pool.getName())); } - } catch (Exception e) { - // com.sun.management may not be available } }); } + /** Returns Attributes carrying jvm.memory.type and jvm.memory.pool.name for the given pool. */ + private static Attributes poolAttributes(MemoryPoolMXBean pool) { + return Attributes.of( + MEMORY_TYPE, pool.getType().name().toLowerCase(), + MEMORY_POOL, pool.getName()); + } + + /** Returns the com.sun.management OperatingSystemMXBean if available, otherwise null. */ + private static OperatingSystemMXBean sunOsBean() { + java.lang.management.OperatingSystemMXBean bean = ManagementFactory.getOperatingSystemMXBean(); + return bean instanceof OperatingSystemMXBean ? (OperatingSystemMXBean) bean : null; + } + private JvmOtlpRuntimeMetrics() {} } From 341de72e82ba5c65c190884f30ee75965650e1bb Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Thu, 7 May 2026 17:44:54 -0400 Subject: [PATCH 14/15] Apply review hardening to JvmOtlpRuntimeMetrics POC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - AtomicBoolean for the started flag so a concurrent second start() loses the compareAndSet. The flag now only flips true after the try block — catch rolls it back so a transient registration failure can retry. - HEAP_ATTRS / NON_HEAP_ATTRS constants replace 8 inline allocations of the same Attributes object. - Locale.ROOT on pool.getType().name().toLowerCase() to avoid Turkish- locale i18n footguns. - Gate startOtlpRuntimeMetrics on isMetricsOtlpExporterEnabled too — if OTEL_METRICS_EXPORTER=none, OtlpMetricsService doesn't start, so registering callbacks would just queue measurements with nowhere to go. Default OTEL_METRICS_EXPORTER is "otlp", so the happy path is unchanged. - reflect-config.json: register the ctor alongside start() to match the shape of the other entries in the same file. - Test cleanup: assertFalse(empty) instead of assertTrue(!empty), import the previously fully-qualified java.util.* names. --- .../java/datadog/trace/bootstrap/Agent.java | 16 ++++----- .../dd-java-agent/reflect-config.json | 1 + .../shim/metrics/JvmOtlpRuntimeMetrics.java | 36 +++++++++---------- .../metrics/JvmOtlpRuntimeMetricsTest.java | 18 ++++++---- 4 files changed, 37 insertions(+), 34 deletions(-) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index d5c2290055b..1fa98a07ac6 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -847,14 +847,14 @@ private static synchronized void installDatadogTracer( initTelemetry.onFatalError(ex); } - // OTLP runtime metrics start unconditionally when both DD_METRICS_OTEL_ENABLED - // and DD_RUNTIME_METRICS_ENABLED are set, regardless of whether the application - // imports the OTel API. The OTLP exporter (OtlpMetricsService) was just started - // by CoreTracer above; this registers the JVM metric callbacks with - // OtelMeterProvider so the periodic export has data to collect. Done here - // (not in startJmx, which is delayed 15s) so callbacks are registered before - // the exporter's first scheduled flush — otherwise short-lived tests miss data. - if (Config.get().isRuntimeMetricsEnabled() && InstrumenterConfig.get().isMetricsOtelEnabled()) { + // Register JVM runtime metric callbacks against the OtelMeterProvider after + // CoreTracer has started OtlpMetricsService. Skip when OTEL_METRICS_EXPORTER=none + // since there's no exporter to collect against. Done here (not in the delayed + // startJmx) so callbacks are in place before the exporter's first flush. + Config cfg = Config.get(); + if (cfg.isRuntimeMetricsEnabled() + && InstrumenterConfig.get().isMetricsOtelEnabled() + && cfg.isMetricsOtlpExporterEnabled()) { startOtlpRuntimeMetrics(); } diff --git a/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json b/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json index a3470425d35..2865cfad830 100644 --- a/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json +++ b/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json @@ -222,6 +222,7 @@ { "name": "datadog.trace.bootstrap.otel.shim.metrics.JvmOtlpRuntimeMetrics", "methods": [ + {"name": "", "parameterTypes": []}, {"name": "start", "parameterTypes": []} ] } diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java index 1913f206ebb..0c50894a8a3 100644 --- a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java +++ b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java @@ -11,6 +11,8 @@ import java.lang.management.MemoryUsage; import java.lang.management.ThreadMXBean; import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.ToLongFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,15 +30,16 @@ public final class JvmOtlpRuntimeMetrics { AttributeKey.stringKey("jvm.memory.pool.name"); private static final AttributeKey BUFFER_POOL = AttributeKey.stringKey("jvm.buffer.pool.name"); + private static final Attributes HEAP_ATTRS = Attributes.of(MEMORY_TYPE, "heap"); + private static final Attributes NON_HEAP_ATTRS = Attributes.of(MEMORY_TYPE, "non_heap"); - private static volatile boolean started = false; + private static final AtomicBoolean started = new AtomicBoolean(false); /** Registers all JVM runtime metric instruments on the OTel MeterProvider. */ public static void start() { - if (started) { + if (!started.compareAndSet(false, true)) { return; } - started = true; try { Meter meter = OtelMeterProvider.INSTANCE.get(INSTRUMENTATION_SCOPE); @@ -47,6 +50,8 @@ public static void start() { registerCpuMetrics(meter); log.debug("Started OTLP runtime metrics with OTel-native naming (jvm.*)"); } catch (Exception e) { + // Roll back so a subsequent retry can re-register if the failure was transient. + started.set(false); log.error("Failed to start JVM OTLP runtime metrics", e); } } @@ -67,11 +72,8 @@ private static void registerMemoryMetrics(Meter meter) { .setUnit("By") .buildWithCallback( measurement -> { - measurement.record( - memoryBean.getHeapMemoryUsage().getUsed(), Attributes.of(MEMORY_TYPE, "heap")); - measurement.record( - memoryBean.getNonHeapMemoryUsage().getUsed(), - Attributes.of(MEMORY_TYPE, "non_heap")); + measurement.record(memoryBean.getHeapMemoryUsage().getUsed(), HEAP_ATTRS); + measurement.record(memoryBean.getNonHeapMemoryUsage().getUsed(), NON_HEAP_ATTRS); for (MemoryPoolMXBean pool : pools) { measurement.record(pool.getUsage().getUsed(), poolAttributes(pool)); } @@ -83,12 +85,8 @@ private static void registerMemoryMetrics(Meter meter) { .setUnit("By") .buildWithCallback( measurement -> { - measurement.record( - memoryBean.getHeapMemoryUsage().getCommitted(), - Attributes.of(MEMORY_TYPE, "heap")); - measurement.record( - memoryBean.getNonHeapMemoryUsage().getCommitted(), - Attributes.of(MEMORY_TYPE, "non_heap")); + measurement.record(memoryBean.getHeapMemoryUsage().getCommitted(), HEAP_ATTRS); + measurement.record(memoryBean.getNonHeapMemoryUsage().getCommitted(), NON_HEAP_ATTRS); for (MemoryPoolMXBean pool : pools) { measurement.record(pool.getUsage().getCommitted(), poolAttributes(pool)); } @@ -102,11 +100,11 @@ private static void registerMemoryMetrics(Meter meter) { measurement -> { long heapMax = memoryBean.getHeapMemoryUsage().getMax(); if (heapMax > 0) { - measurement.record(heapMax, Attributes.of(MEMORY_TYPE, "heap")); + measurement.record(heapMax, HEAP_ATTRS); } long nonHeapMax = memoryBean.getNonHeapMemoryUsage().getMax(); if (nonHeapMax > 0) { - measurement.record(nonHeapMax, Attributes.of(MEMORY_TYPE, "non_heap")); + measurement.record(nonHeapMax, NON_HEAP_ATTRS); } for (MemoryPoolMXBean pool : pools) { long max = pool.getUsage().getMax(); @@ -124,11 +122,11 @@ private static void registerMemoryMetrics(Meter meter) { measurement -> { long heapInit = memoryBean.getHeapMemoryUsage().getInit(); if (heapInit > 0) { - measurement.record(heapInit, Attributes.of(MEMORY_TYPE, "heap")); + measurement.record(heapInit, HEAP_ATTRS); } long nonHeapInit = memoryBean.getNonHeapMemoryUsage().getInit(); if (nonHeapInit > 0) { - measurement.record(nonHeapInit, Attributes.of(MEMORY_TYPE, "non_heap")); + measurement.record(nonHeapInit, NON_HEAP_ATTRS); } }); @@ -292,7 +290,7 @@ private static void bufferPoolMetric( /** Returns Attributes carrying jvm.memory.type and jvm.memory.pool.name for the given pool. */ private static Attributes poolAttributes(MemoryPoolMXBean pool) { return Attributes.of( - MEMORY_TYPE, pool.getType().name().toLowerCase(), + MEMORY_TYPE, pool.getType().name().toLowerCase(Locale.ROOT), MEMORY_POOL, pool.getName()); } diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java index f447a44601c..e4474321c62 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java @@ -1,6 +1,7 @@ package opentelemetry147.metrics; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -14,13 +15,16 @@ import datadog.trace.bootstrap.otlp.metrics.OtlpMetricVisitor; import datadog.trace.bootstrap.otlp.metrics.OtlpMetricsVisitor; import datadog.trace.bootstrap.otlp.metrics.OtlpScopedMetricsVisitor; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.TreeSet; import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -69,10 +73,10 @@ void registersExpectedJvmMetrics() { for (String metric : expectedMetrics) { assertTrue( names.contains(metric), - "Expected metric '" + metric + "' not found. Got: " + new java.util.TreeSet<>(names)); + "Expected metric '" + metric + "' not found. Got: " + new TreeSet<>(names)); } - assertEquals(15, names.size(), "Expected 15 metrics, got: " + new java.util.TreeSet<>(names)); + assertEquals(15, names.size(), "Expected 15 metrics, got: " + new TreeSet<>(names)); // No DD-proprietary names should be present List ddNames = @@ -103,7 +107,7 @@ void jvmMemoryUsedHeapValueIsPositive() { points.stream() .filter(p -> "heap".equals(p.attrs.get("jvm.memory.type"))) .collect(Collectors.toList()); - assertTrue(!heapPoints.isEmpty(), "jvm.memory.used should have heap data point"); + assertFalse(heapPoints.isEmpty(), "jvm.memory.used should have heap data point"); assertTrue( heapPoints.get(0).value.longValue() > 0, "jvm.memory.used heap value should be positive, got " + heapPoints.get(0).value); @@ -116,7 +120,7 @@ void jvmThreadCountIsPositive() { List threadPoints = collector.points.get("jvm.thread.count"); assertNotNull(threadPoints, "jvm.thread.count should have data points"); - assertTrue(!threadPoints.isEmpty(), "jvm.thread.count should have data points"); + assertFalse(threadPoints.isEmpty(), "jvm.thread.count should have data points"); assertTrue( threadPoints.get(0).value.longValue() > 0, "jvm.thread.count value should be positive, got " + threadPoints.get(0).value); @@ -149,7 +153,7 @@ public OtlpScopedMetricsVisitor visitScopedMetrics(OtelInstrumentationScope scop public OtlpMetricVisitor visitMetric(OtelInstrumentDescriptor descriptor) { currentInstrument = descriptor.getName().toString(); metricNames.add(currentInstrument); - points.computeIfAbsent(currentInstrument, k -> new java.util.ArrayList<>()); + points.computeIfAbsent(currentInstrument, k -> new ArrayList<>()); return this; } @@ -169,7 +173,7 @@ public void visitDataPoint(OtlpDataPoint point) { value = ((OtlpDoublePoint) point).value; } points - .computeIfAbsent(currentInstrument, k -> new java.util.ArrayList<>()) + .computeIfAbsent(currentInstrument, k -> new ArrayList<>()) .add(new DataPointEntry(attrs, value)); } @@ -180,7 +184,7 @@ Set attributeValues(String metricName, String attrKey) { } return entries.stream() .map(e -> e.attrs.get(attrKey)) - .filter(java.util.Objects::nonNull) + .filter(Objects::nonNull) .map(Object::toString) .collect(Collectors.toCollection(LinkedHashSet::new)); } From cd8640cfdaa86fdf59ea410abc22fa39cb329364 Mon Sep 17 00:00:00 2001 From: Maximo Bautista Date: Thu, 7 May 2026 18:09:12 -0400 Subject: [PATCH 15/15] Cache MXBeans and drop start() rollback in JvmOtlpRuntimeMetrics - Cache ClassLoadingMXBean and OperatingSystemMXBean once at registration - Gate CPU metric registration on com.sun OperatingSystemMXBean availability - Drop started.set(false) rollback: partial registration is worse than no retry - Add startIsIdempotent test Co-Authored-By: Claude Opus 4.7 (1M context) --- .../shim/metrics/JvmOtlpRuntimeMetrics.java | 85 ++++++++----------- .../metrics/JvmOtlpRuntimeMetricsTest.java | 17 ++++ 2 files changed, 53 insertions(+), 49 deletions(-) diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java index 0c50894a8a3..6b151ae7370 100644 --- a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java +++ b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java @@ -5,6 +5,7 @@ import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.Meter; import java.lang.management.BufferPoolMXBean; +import java.lang.management.ClassLoadingMXBean; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; import java.lang.management.MemoryPoolMXBean; @@ -50,8 +51,6 @@ public static void start() { registerCpuMetrics(meter); log.debug("Started OTLP runtime metrics with OTel-native naming (jvm.*)"); } catch (Exception e) { - // Roll back so a subsequent retry can re-register if the failure was transient. - started.set(false); log.error("Failed to start JVM OTLP runtime metrics", e); } } @@ -187,32 +186,27 @@ private static void registerThreadMetrics(Meter meter) { * Stable per spec. */ private static void registerClassLoadingMetrics(Meter meter) { + ClassLoadingMXBean classLoadingBean = ManagementFactory.getClassLoadingMXBean(); meter .counterBuilder("jvm.class.loaded") .setDescription("Number of classes loaded since JVM start.") .setUnit("{class}") .buildWithCallback( - measurement -> - measurement.record( - ManagementFactory.getClassLoadingMXBean().getTotalLoadedClassCount())); + measurement -> measurement.record(classLoadingBean.getTotalLoadedClassCount())); meter .upDownCounterBuilder("jvm.class.count") .setDescription("Number of classes currently loaded.") .setUnit("{class}") .buildWithCallback( - measurement -> - measurement.record( - ManagementFactory.getClassLoadingMXBean().getLoadedClassCount())); + measurement -> measurement.record(classLoadingBean.getLoadedClassCount())); meter .counterBuilder("jvm.class.unloaded") .setDescription("Number of classes unloaded since JVM start.") .setUnit("{class}") .buildWithCallback( - measurement -> - measurement.record( - ManagementFactory.getClassLoadingMXBean().getUnloadedClassCount())); + measurement -> measurement.record(classLoadingBean.getUnloadedClassCount())); } /** @@ -220,22 +214,37 @@ private static void registerClassLoadingMetrics(Meter meter) { * Stable per spec. */ private static void registerCpuMetrics(Meter meter) { - meter - .counterBuilder("jvm.cpu.time") - .ofDoubles() - .setDescription("CPU time used by the process as reported by the JVM.") - .setUnit("s") - .buildWithCallback( - measurement -> { - OperatingSystemMXBean osBean = sunOsBean(); - if (osBean == null) { - return; - } - long nanos = osBean.getProcessCpuTime(); - if (nanos >= 0) { - measurement.record(nanos / 1e9); - } - }); + java.lang.management.OperatingSystemMXBean rawOsBean = + ManagementFactory.getOperatingSystemMXBean(); + OperatingSystemMXBean osBean = + rawOsBean instanceof OperatingSystemMXBean ? (OperatingSystemMXBean) rawOsBean : null; + + if (osBean != null) { + meter + .counterBuilder("jvm.cpu.time") + .ofDoubles() + .setDescription("CPU time used by the process as reported by the JVM.") + .setUnit("s") + .buildWithCallback( + measurement -> { + long nanos = osBean.getProcessCpuTime(); + if (nanos >= 0) { + measurement.record(nanos / 1e9); + } + }); + + meter + .gaugeBuilder("jvm.cpu.recent_utilization") + .setDescription("Recent CPU utilization for the process as reported by the JVM.") + .setUnit("1") + .buildWithCallback( + measurement -> { + double cpuLoad = osBean.getProcessCpuLoad(); + if (cpuLoad >= 0) { + measurement.record(cpuLoad); + } + }); + } meter .upDownCounterBuilder("jvm.cpu.count") @@ -243,22 +252,6 @@ private static void registerCpuMetrics(Meter meter) { .setUnit("{cpu}") .buildWithCallback( measurement -> measurement.record(Runtime.getRuntime().availableProcessors())); - - meter - .gaugeBuilder("jvm.cpu.recent_utilization") - .setDescription("Recent CPU utilization for the process as reported by the JVM.") - .setUnit("1") - .buildWithCallback( - measurement -> { - OperatingSystemMXBean osBean = sunOsBean(); - if (osBean == null) { - return; - } - double cpuLoad = osBean.getProcessCpuLoad(); - if (cpuLoad >= 0) { - measurement.record(cpuLoad); - } - }); } /** @@ -294,11 +287,5 @@ private static Attributes poolAttributes(MemoryPoolMXBean pool) { MEMORY_POOL, pool.getName()); } - /** Returns the com.sun.management OperatingSystemMXBean if available, otherwise null. */ - private static OperatingSystemMXBean sunOsBean() { - java.lang.management.OperatingSystemMXBean bean = ManagementFactory.getOperatingSystemMXBean(); - return bean instanceof OperatingSystemMXBean ? (OperatingSystemMXBean) bean : null; - } - private JvmOtlpRuntimeMetrics() {} } diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java index e4474321c62..72ddfd3c840 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java @@ -126,6 +126,23 @@ void jvmThreadCountIsPositive() { "jvm.thread.count value should be positive, got " + threadPoints.get(0).value); } + @Test + void startIsIdempotent() { + MetricCollector before = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(before); + int countBefore = before.metricNames.size(); + + JvmOtlpRuntimeMetrics.start(); + JvmOtlpRuntimeMetrics.start(); + + MetricCollector after = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(after); + assertEquals( + countBefore, + after.metricNames.size(), + "Repeated start() must not register duplicate instruments"); + } + static final class DataPointEntry { final Map attrs; final Number value;