diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 5288f92dbe3..1fa98a07ac6 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -26,6 +26,7 @@ import datadog.instrument.utils.ClassLoaderValue; import datadog.metrics.api.statsd.StatsDClientManager; import datadog.trace.api.Config; +import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.Platform; import datadog.trace.api.WithGlobalTracer; import datadog.trace.api.appsec.AppSecEventTracker; @@ -846,6 +847,17 @@ private static synchronized void installDatadogTracer( initTelemetry.onFatalError(ex); } + // Register JVM runtime metric callbacks against the OtelMeterProvider after + // CoreTracer has started OtlpMetricsService. Skip when OTEL_METRICS_EXPORTER=none + // since there's no exporter to collect against. Done here (not in the delayed + // startJmx) so callbacks are in place before the exporter's first flush. + Config cfg = Config.get(); + if (cfg.isRuntimeMetricsEnabled() + && InstrumenterConfig.get().isMetricsOtelEnabled() + && cfg.isMetricsOtlpExporterEnabled()) { + startOtlpRuntimeMetrics(); + } + StaticEventLogger.end("GlobalTracer"); } @@ -989,6 +1001,27 @@ private static synchronized void initializeJmxSystemAccessProvider( } } + /** + * Registers OTLP runtime metric callbacks (JVM heap, CPU, threads, classes, etc.) with the + * agent's OtelMeterProvider. The periodic OTLP exporter started by CoreTracer then collects and + * exports them — this is the same pattern Node and .NET use to start their runtime metrics + * unconditionally during tracer init, independent of any app-side OTel API usage. + */ + private static synchronized void startOtlpRuntimeMetrics() { + final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(AGENT_CLASSLOADER); + final Class jvmOtlpClass = + AGENT_CLASSLOADER.loadClass("datadog.opentelemetry.shim.metrics.JvmOtlpRuntimeMetrics"); + final Method startMethod = jvmOtlpClass.getMethod("start"); + startMethod.invoke(null); + } catch (final Throwable ex) { + log.error("Throwable thrown while starting OTLP runtime metrics", ex); + } finally { + safelySetContextClassLoader(contextLoader); + } + } + private static synchronized void startJmxFetch() { final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); try { diff --git a/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json b/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json index 3225d6dd59f..2865cfad830 100644 --- a/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json +++ b/dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json @@ -218,5 +218,12 @@ "methods": [ {"name": "", "parameterTypes": []} ] + }, + { + "name": "datadog.trace.bootstrap.otel.shim.metrics.JvmOtlpRuntimeMetrics", + "methods": [ + {"name": "", "parameterTypes": []}, + {"name": "start", "parameterTypes": []} + ] } ] diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java new file mode 100644 index 00000000000..6b151ae7370 --- /dev/null +++ b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/JvmOtlpRuntimeMetrics.java @@ -0,0 +1,291 @@ +package datadog.opentelemetry.shim.metrics; + +import com.sun.management.OperatingSystemMXBean; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.Meter; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ClassLoadingMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryUsage; +import java.lang.management.ThreadMXBean; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.ToLongFunction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Registers JVM runtime metrics with OTel-native names against the agent's MeterProvider. See + * https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/. + */ +public final class JvmOtlpRuntimeMetrics { + + private static final Logger log = LoggerFactory.getLogger(JvmOtlpRuntimeMetrics.class); + private static final String INSTRUMENTATION_SCOPE = "datadog.jvm.runtime"; + private static final AttributeKey MEMORY_TYPE = AttributeKey.stringKey("jvm.memory.type"); + private static final AttributeKey MEMORY_POOL = + AttributeKey.stringKey("jvm.memory.pool.name"); + private static final AttributeKey BUFFER_POOL = + AttributeKey.stringKey("jvm.buffer.pool.name"); + private static final Attributes HEAP_ATTRS = Attributes.of(MEMORY_TYPE, "heap"); + private static final Attributes NON_HEAP_ATTRS = Attributes.of(MEMORY_TYPE, "non_heap"); + + private static final AtomicBoolean started = new AtomicBoolean(false); + + /** Registers all JVM runtime metric instruments on the OTel MeterProvider. */ + public static void start() { + if (!started.compareAndSet(false, true)) { + return; + } + + try { + Meter meter = OtelMeterProvider.INSTANCE.get(INSTRUMENTATION_SCOPE); + registerMemoryMetrics(meter); + registerBufferMetrics(meter); + registerThreadMetrics(meter); + registerClassLoadingMetrics(meter); + registerCpuMetrics(meter); + log.debug("Started OTLP runtime metrics with OTel-native naming (jvm.*)"); + } catch (Exception e) { + log.error("Failed to start JVM OTLP runtime metrics", e); + } + } + + // jvm.gc.duration is excluded — spec requires Histogram, JMX only exposes cumulative time. + + /** + * jvm.memory.used, jvm.memory.committed, jvm.memory.limit, jvm.memory.init, + * jvm.memory.used_after_last_gc — all UpDownCounter per spec. + */ + private static void registerMemoryMetrics(Meter meter) { + MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); + List pools = ManagementFactory.getMemoryPoolMXBeans(); + + meter + .upDownCounterBuilder("jvm.memory.used") + .setDescription("Measure of memory used.") + .setUnit("By") + .buildWithCallback( + measurement -> { + measurement.record(memoryBean.getHeapMemoryUsage().getUsed(), HEAP_ATTRS); + measurement.record(memoryBean.getNonHeapMemoryUsage().getUsed(), NON_HEAP_ATTRS); + for (MemoryPoolMXBean pool : pools) { + measurement.record(pool.getUsage().getUsed(), poolAttributes(pool)); + } + }); + + meter + .upDownCounterBuilder("jvm.memory.committed") + .setDescription("Measure of memory committed.") + .setUnit("By") + .buildWithCallback( + measurement -> { + measurement.record(memoryBean.getHeapMemoryUsage().getCommitted(), HEAP_ATTRS); + measurement.record(memoryBean.getNonHeapMemoryUsage().getCommitted(), NON_HEAP_ATTRS); + for (MemoryPoolMXBean pool : pools) { + measurement.record(pool.getUsage().getCommitted(), poolAttributes(pool)); + } + }); + + meter + .upDownCounterBuilder("jvm.memory.limit") + .setDescription("Measure of max obtainable memory.") + .setUnit("By") + .buildWithCallback( + measurement -> { + long heapMax = memoryBean.getHeapMemoryUsage().getMax(); + if (heapMax > 0) { + measurement.record(heapMax, HEAP_ATTRS); + } + long nonHeapMax = memoryBean.getNonHeapMemoryUsage().getMax(); + if (nonHeapMax > 0) { + measurement.record(nonHeapMax, NON_HEAP_ATTRS); + } + for (MemoryPoolMXBean pool : pools) { + long max = pool.getUsage().getMax(); + if (max > 0) { + measurement.record(max, poolAttributes(pool)); + } + } + }); + + meter + .upDownCounterBuilder("jvm.memory.init") + .setDescription("Measure of initial memory requested.") + .setUnit("By") + .buildWithCallback( + measurement -> { + long heapInit = memoryBean.getHeapMemoryUsage().getInit(); + if (heapInit > 0) { + measurement.record(heapInit, HEAP_ATTRS); + } + long nonHeapInit = memoryBean.getNonHeapMemoryUsage().getInit(); + if (nonHeapInit > 0) { + measurement.record(nonHeapInit, NON_HEAP_ATTRS); + } + }); + + meter + .upDownCounterBuilder("jvm.memory.used_after_last_gc") + .setDescription("Measure of memory used after the most recent garbage collection event.") + .setUnit("By") + .buildWithCallback( + measurement -> { + for (MemoryPoolMXBean pool : pools) { + MemoryUsage collectionUsage = pool.getCollectionUsage(); + if (collectionUsage != null && collectionUsage.getUsed() >= 0) { + measurement.record(collectionUsage.getUsed(), poolAttributes(pool)); + } + } + }); + } + + /** jvm.buffer.* (UpDownCounter, Development) — direct + mapped pool metrics. */ + private static void registerBufferMetrics(Meter meter) { + List bufferPools = + ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); + bufferPoolMetric( + meter, + "jvm.buffer.memory.used", + "Measure of memory used by buffers.", + "By", + bufferPools, + BufferPoolMXBean::getMemoryUsed); + bufferPoolMetric( + meter, + "jvm.buffer.memory.limit", + "Measure of total memory capacity of buffers.", + "By", + bufferPools, + BufferPoolMXBean::getTotalCapacity); + bufferPoolMetric( + meter, + "jvm.buffer.count", + "Number of buffers in the pool.", + "{buffer}", + bufferPools, + BufferPoolMXBean::getCount); + } + + /** jvm.thread.count (UpDownCounter, Stable). */ + private static void registerThreadMetrics(Meter meter) { + ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); + meter + .upDownCounterBuilder("jvm.thread.count") + .setDescription("Number of executing platform threads.") + .setUnit("{thread}") + .buildWithCallback(measurement -> measurement.record(threadBean.getThreadCount())); + } + + /** + * jvm.class.loaded (Counter), jvm.class.unloaded (Counter), jvm.class.count (UpDownCounter) — all + * Stable per spec. + */ + private static void registerClassLoadingMetrics(Meter meter) { + ClassLoadingMXBean classLoadingBean = ManagementFactory.getClassLoadingMXBean(); + meter + .counterBuilder("jvm.class.loaded") + .setDescription("Number of classes loaded since JVM start.") + .setUnit("{class}") + .buildWithCallback( + measurement -> measurement.record(classLoadingBean.getTotalLoadedClassCount())); + + meter + .upDownCounterBuilder("jvm.class.count") + .setDescription("Number of classes currently loaded.") + .setUnit("{class}") + .buildWithCallback( + measurement -> measurement.record(classLoadingBean.getLoadedClassCount())); + + meter + .counterBuilder("jvm.class.unloaded") + .setDescription("Number of classes unloaded since JVM start.") + .setUnit("{class}") + .buildWithCallback( + measurement -> measurement.record(classLoadingBean.getUnloadedClassCount())); + } + + /** + * jvm.cpu.time (Counter), jvm.cpu.count (UpDownCounter), jvm.cpu.recent_utilization (Gauge) — all + * Stable per spec. + */ + private static void registerCpuMetrics(Meter meter) { + java.lang.management.OperatingSystemMXBean rawOsBean = + ManagementFactory.getOperatingSystemMXBean(); + OperatingSystemMXBean osBean = + rawOsBean instanceof OperatingSystemMXBean ? (OperatingSystemMXBean) rawOsBean : null; + + if (osBean != null) { + meter + .counterBuilder("jvm.cpu.time") + .ofDoubles() + .setDescription("CPU time used by the process as reported by the JVM.") + .setUnit("s") + .buildWithCallback( + measurement -> { + long nanos = osBean.getProcessCpuTime(); + if (nanos >= 0) { + measurement.record(nanos / 1e9); + } + }); + + meter + .gaugeBuilder("jvm.cpu.recent_utilization") + .setDescription("Recent CPU utilization for the process as reported by the JVM.") + .setUnit("1") + .buildWithCallback( + measurement -> { + double cpuLoad = osBean.getProcessCpuLoad(); + if (cpuLoad >= 0) { + measurement.record(cpuLoad); + } + }); + } + + meter + .upDownCounterBuilder("jvm.cpu.count") + .setDescription("Number of processors available to the JVM.") + .setUnit("{cpu}") + .buildWithCallback( + measurement -> measurement.record(Runtime.getRuntime().availableProcessors())); + } + + /** + * Builds an UpDownCounter that iterates each platform buffer pool and records {@code getter} with + * the {@code jvm.buffer.pool.name} attribute. Skips negative readings. + */ + private static void bufferPoolMetric( + Meter meter, + String name, + String description, + String unit, + List bufferPools, + ToLongFunction getter) { + meter + .upDownCounterBuilder(name) + .setDescription(description) + .setUnit(unit) + .buildWithCallback( + measurement -> { + for (BufferPoolMXBean pool : bufferPools) { + long value = getter.applyAsLong(pool); + if (value >= 0) { + measurement.record(value, Attributes.of(BUFFER_POOL, pool.getName())); + } + } + }); + } + + /** Returns Attributes carrying jvm.memory.type and jvm.memory.pool.name for the given pool. */ + private static Attributes poolAttributes(MemoryPoolMXBean pool) { + return Attributes.of( + MEMORY_TYPE, pool.getType().name().toLowerCase(Locale.ROOT), + MEMORY_POOL, pool.getName()); + } + + private JvmOtlpRuntimeMetrics() {} +} diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java new file mode 100644 index 00000000000..72ddfd3c840 --- /dev/null +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/src/test/java/opentelemetry147/metrics/JvmOtlpRuntimeMetricsTest.java @@ -0,0 +1,209 @@ +package opentelemetry147.metrics; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.opentelemetry.shim.metrics.JvmOtlpRuntimeMetrics; +import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; +import datadog.trace.bootstrap.otel.metrics.OtelInstrumentDescriptor; +import datadog.trace.bootstrap.otel.metrics.data.OtelMetricRegistry; +import datadog.trace.bootstrap.otlp.metrics.OtlpDataPoint; +import datadog.trace.bootstrap.otlp.metrics.OtlpDoublePoint; +import datadog.trace.bootstrap.otlp.metrics.OtlpLongPoint; +import datadog.trace.bootstrap.otlp.metrics.OtlpMetricVisitor; +import datadog.trace.bootstrap.otlp.metrics.OtlpMetricsVisitor; +import datadog.trace.bootstrap.otlp.metrics.OtlpScopedMetricsVisitor; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Tests that JVM runtime metrics are registered and exported via OTLP using OTel semantic + * convention names (jvm.memory.used, jvm.thread.count, etc.). + * + *

Ref: https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ + * + *

Ref: + * https://github.com/DataDog/semantic-core/blob/main/sor/domains/metrics/integrations/java/_equivalence/ + */ +public class JvmOtlpRuntimeMetricsTest { + + @BeforeAll + static void setUp() { + System.setProperty("dd.metrics.otel.enabled", "true"); + JvmOtlpRuntimeMetrics.start(); + } + + @Test + void registersExpectedJvmMetrics() { + MetricCollector collector = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(collector); + + List expectedMetrics = + Arrays.asList( + "jvm.memory.used", + "jvm.memory.committed", + "jvm.memory.limit", + "jvm.memory.init", + "jvm.memory.used_after_last_gc", + "jvm.buffer.memory.used", + "jvm.buffer.memory.limit", + "jvm.buffer.count", + "jvm.thread.count", + "jvm.class.loaded", + "jvm.class.count", + "jvm.class.unloaded", + "jvm.cpu.time", + "jvm.cpu.count", + "jvm.cpu.recent_utilization"); + + Set names = collector.metricNames; + for (String metric : expectedMetrics) { + assertTrue( + names.contains(metric), + "Expected metric '" + metric + "' not found. Got: " + new TreeSet<>(names)); + } + + assertEquals(15, names.size(), "Expected 15 metrics, got: " + new TreeSet<>(names)); + + // No DD-proprietary names should be present + List ddNames = + names.stream() + .filter(n -> n.startsWith("jvm.heap_memory") || n.startsWith("jvm.thread_count")) + .collect(Collectors.toList()); + assertTrue(ddNames.isEmpty(), "DD-proprietary names leaked: " + ddNames); + } + + @Test + void jvmMemoryUsedHasHeapAndNonHeapTypeAttributes() { + MetricCollector collector = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(collector); + + Set types = collector.attributeValues("jvm.memory.used", "jvm.memory.type"); + assertTrue(types.contains("heap"), "jvm.memory.used should have heap attribute"); + assertTrue(types.contains("non_heap"), "jvm.memory.used should have non_heap attribute"); + } + + @Test + void jvmMemoryUsedHeapValueIsPositive() { + MetricCollector collector = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(collector); + + List points = collector.points.get("jvm.memory.used"); + assertNotNull(points, "jvm.memory.used should have data points"); + List heapPoints = + points.stream() + .filter(p -> "heap".equals(p.attrs.get("jvm.memory.type"))) + .collect(Collectors.toList()); + assertFalse(heapPoints.isEmpty(), "jvm.memory.used should have heap data point"); + assertTrue( + heapPoints.get(0).value.longValue() > 0, + "jvm.memory.used heap value should be positive, got " + heapPoints.get(0).value); + } + + @Test + void jvmThreadCountIsPositive() { + MetricCollector collector = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(collector); + + List threadPoints = collector.points.get("jvm.thread.count"); + assertNotNull(threadPoints, "jvm.thread.count should have data points"); + assertFalse(threadPoints.isEmpty(), "jvm.thread.count should have data points"); + assertTrue( + threadPoints.get(0).value.longValue() > 0, + "jvm.thread.count value should be positive, got " + threadPoints.get(0).value); + } + + @Test + void startIsIdempotent() { + MetricCollector before = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(before); + int countBefore = before.metricNames.size(); + + JvmOtlpRuntimeMetrics.start(); + JvmOtlpRuntimeMetrics.start(); + + MetricCollector after = new MetricCollector(); + OtelMetricRegistry.INSTANCE.collectMetrics(after); + assertEquals( + countBefore, + after.metricNames.size(), + "Repeated start() must not register duplicate instruments"); + } + + static final class DataPointEntry { + final Map attrs; + final Number value; + + DataPointEntry(Map attrs, Number value) { + this.attrs = attrs; + this.value = value; + } + } + + static final class MetricCollector + implements OtlpMetricsVisitor, OtlpScopedMetricsVisitor, OtlpMetricVisitor { + + String currentInstrument = ""; + final Map currentAttrs = new LinkedHashMap<>(); + final Set metricNames = new LinkedHashSet<>(); + final Map> points = new LinkedHashMap<>(); + + @Override + public OtlpScopedMetricsVisitor visitScopedMetrics(OtelInstrumentationScope scope) { + return this; + } + + @Override + public OtlpMetricVisitor visitMetric(OtelInstrumentDescriptor descriptor) { + currentInstrument = descriptor.getName().toString(); + metricNames.add(currentInstrument); + points.computeIfAbsent(currentInstrument, k -> new ArrayList<>()); + return this; + } + + @Override + public void visitAttribute(int type, String key, Object value) { + currentAttrs.put(key, value == null ? null : value.toString()); + } + + @Override + public void visitDataPoint(OtlpDataPoint point) { + Map attrs = new HashMap<>(currentAttrs); + currentAttrs.clear(); + Number value = 0; + if (point instanceof OtlpLongPoint) { + value = ((OtlpLongPoint) point).value; + } else if (point instanceof OtlpDoublePoint) { + value = ((OtlpDoublePoint) point).value; + } + points + .computeIfAbsent(currentInstrument, k -> new ArrayList<>()) + .add(new DataPointEntry(attrs, value)); + } + + Set attributeValues(String metricName, String attrKey) { + List entries = points.get(metricName); + if (entries == null) { + return new LinkedHashSet<>(); + } + return entries.stream() + .map(e -> e.attrs.get(attrKey)) + .filter(Objects::nonNull) + .map(Object::toString) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + } +}