feat: add trace stats computation

.github/CODEOWNERS

@@ -0,0 +1,4 @@
+* @DataDog/serverless-azure-gcp @DataDog/serverless-aws
+
+crates/datadog-serverless-compat/ @DataDog/serverless-azure-gcp
+crates/datadog-trace-agent/ @DataDog/serverless-azure-gcp

LICENSE-3rdparty.csv

@@ -3,6 +3,7 @@ addr2line,https://github.com/gimli-rs/addr2line,Apache-2.0 OR MIT,The addr2line
 adler2,https://github.com/oyvindln/adler2,0BSD OR MIT OR Apache-2.0,"Jonas Schievink <jonasschievink@gmail.com>, oyvindln <oyvindln@users.noreply.github.com>"
 ahash,https://github.com/tkaitchuck/ahash,MIT OR Apache-2.0,Tom Kaitchuck <Tom.Kaitchuck@gmail.com>
 aho-corasick,https://github.com/BurntSushi/aho-corasick,Unlicense OR MIT,Andrew Gallant <jamslam@gmail.com>
+allocator-api2,https://github.com/zakarumych/allocator-api2,MIT OR Apache-2.0,Zakarum <zaq.dev@icloud.com>
 anyhow,https://github.com/dtolnay/anyhow,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
 assert-json-diff,https://github.com/davidpdrsn/assert-json-diff,MIT,David Pedersen <david.pdrsn@gmail.com>
 async-lock,https://github.com/smol-rs/async-lock,Apache-2.0 OR MIT,Stjepan Glavina <stjepang@gmail.com>
@@ -54,6 +55,7 @@ fixedbitset,https://github.com/petgraph/fixedbitset,MIT OR Apache-2.0,bluss
 flate2,https://github.com/rust-lang/flate2-rs,MIT OR Apache-2.0,"Alex Crichton <alex@alexcrichton.com>, Josh Triplett <josh@joshtriplett.org>"
 float-cmp,https://github.com/mikedilger/float-cmp,MIT,Mike Dilger <mike@mikedilger.com>
 fnv,https://github.com/servo/rust-fnv,Apache-2.0  OR  MIT,Alex Crichton <alex@alexcrichton.com>
+foldhash,https://github.com/orlp/foldhash,Zlib,Orson Peters <orsonpeters@gmail.com>
 form_urlencoded,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers
 futures,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures Authors
 futures-channel,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-channel Authors
@@ -109,10 +111,12 @@ lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2.
 lazycell,https://github.com/indiv0/lazycell,MIT OR Apache-2.0,"Alex Crichton <alex@alexcrichton.com>, Nikita Pekin <contact@nikitapek.in>"
 libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers
 libdd-common,https://github.com/DataDog/libdatadog/tree/main/datadog-common,Apache-2.0,The libdd-common Authors
+libdd-ddsketch,https://github.com/DataDog/libdatadog/tree/main/libdd-ddsketch,Apache-2.0,The libdd-ddsketch Authors
 libdd-tinybytes,https://github.com/DataDog/libdatadog/tree/main/libdd-tinybytes,Apache-2.0,The libdd-tinybytes Authors
 libdd-trace-normalization,https://github.com/DataDog/libdatadog/tree/main/libdd-trace-normalization,Apache-2.0,David Lee <david.lee@datadoghq.com>
 libdd-trace-obfuscation,https://github.com/DataDog/libdatadog/tree/main/libdd-trace-obfuscation,Apache-2.0,Datadog Inc. <info@datadoghq.com>
 libdd-trace-protobuf,https://github.com/DataDog/libdatadog/tree/main/libdd-trace-protobuf,Apache-2.0,The libdd-trace-protobuf Authors
+libdd-trace-stats,https://github.com/DataDog/libdatadog/tree/main/libdd-trace-stats,Apache-2.0,The libdd-trace-stats Authors
 libdd-trace-utils,https://github.com/DataDog/libdatadog/tree/main/libdd-trace-utils,Apache-2.0,The libdd-trace-utils Authors
 libloading,https://github.com/nagisa/rust_libloading,ISC,Simonas Kazlauskas <libloading@kazlauskas.me>
 linux-raw-sys,https://github.com/sunfishcode/linux-raw-sys,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman <dev@sunfishcode.online>

crates/datadog-serverless-compat/Cargo.toml

@@ -7,7 +7,7 @@ description = "Binary to run trace-agent and dogstatsd servers in Serverless env
 
 [dependencies]
 datadog-trace-agent = { path = "../datadog-trace-agent" }
-libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "435107c245112397914935c0f7148a18b91cafc6" }
+libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "774f830c7ffa62122a2cf8f3eae8c64a3ca5a8d7" }
 dogstatsd = { path = "../dogstatsd", default-features = true }
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
 tokio-util = { version = "0.7", default-features = false }

crates/datadog-serverless-compat/src/main.rs

@@ -18,7 +18,8 @@ use zstd::zstd_safe::CompressionLevel;
 
 use datadog_trace_agent::{
     aggregator::TraceAggregator,
-    config, env_verifier, mini_agent, stats_flusher, stats_processor,
+    config, env_verifier, mini_agent, stats_concentrator_service, stats_flusher, stats_generator,
+    stats_processor,
     trace_flusher::{self, TraceFlusher},
     trace_processor,
 };
@@ -73,6 +74,9 @@ pub async fn main() {
     let dd_use_dogstatsd = env::var("DD_USE_DOGSTATSD")
         .map(|val| val.to_lowercase() != "false")
         .unwrap_or(true);
+    let dd_stats_computation_enabled = env::var("DD_STATS_COMPUTATION_ENABLED")
+        .map(|val| val.to_lowercase() != "false")
+        .unwrap_or(true);
     let dd_statsd_metric_namespace: Option<String> = env::var("DD_STATSD_METRIC_NAMESPACE")
         .ok()
         .and_then(|val| parse_metric_namespace(&val));
@@ -100,6 +104,7 @@ pub async fn main() {
 
     #[allow(clippy::expect_used)]
     tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed");
+    info!("Stats test - buffer length: configurable / default 90, bucket duration: 10");
 
     debug!("Logging subsystem enabled");
 
@@ -124,13 +129,30 @@ pub async fn main() {
         Arc::clone(&config),
     ));
 
+    // Initialize stats concentrator service and generator conditionally
+    let (stats_concentrator_handle, stats_generator) = if dd_stats_computation_enabled {
+        info!("Stats computation enabled");
+        let (stats_concentrator_service, stats_concentrator_handle) =
+            stats_concentrator_service::StatsConcentratorService::new(config.clone());
+        tokio::spawn(stats_concentrator_service.run());
+        let stats_generator = Arc::new(stats_generator::StatsGenerator::new(
+            stats_concentrator_handle.clone(),
+        ));
+        (Some(stats_concentrator_handle), Some(stats_generator))
+    } else {
+        info!("Stats computation disabled");
+        (None, None)
+    };
+
     let mini_agent = Box::new(mini_agent::MiniAgent {
         config: Arc::clone(&config),
         env_verifier,
         trace_processor,
         trace_flusher,
         stats_processor,
         stats_flusher,
+        stats_concentrator: stats_concentrator_handle,
+        stats_generator,
     });
 
     tokio::spawn(async move {

crates/datadog-trace-agent/Cargo.toml

@@ -15,23 +15,25 @@ hyper-http-proxy = { version = "1.1.0", default-features = false, features = [
 ] }
 hyper-util = { version = "0.1", features = ["service"] }
 http-body-util = "0.1"
+thiserror = { version = "1.0", default-features = false }
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
 async-trait = "0.1.64"
 tracing = { version = "0.1", default-features = false }
 serde = { version = "1.0.145", features = ["derive"] }
 serde_json = "1.0"
-libdd-common = { git = "https://github.com/DataDog/libdatadog", rev = "435107c245112397914935c0f7148a18b91cafc6" }
-libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "435107c245112397914935c0f7148a18b91cafc6" }
-libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "435107c245112397914935c0f7148a18b91cafc6" }
-libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "435107c245112397914935c0f7148a18b91cafc6", features = [
+libdd-common = { git = "https://github.com/DataDog/libdatadog", rev = "774f830c7ffa62122a2cf8f3eae8c64a3ca5a8d7" }
+libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "774f830c7ffa62122a2cf8f3eae8c64a3ca5a8d7" }
+libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "774f830c7ffa62122a2cf8f3eae8c64a3ca5a8d7" }
+libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "774f830c7ffa62122a2cf8f3eae8c64a3ca5a8d7", features = [
     "mini_agent",
 ] }
+libdd-trace-stats = { git = "https://github.com/DataDog/libdatadog", rev = "774f830c7ffa62122a2cf8f3eae8c64a3ca5a8d7" }
 
 [dev-dependencies]
 rmp-serde = "1.1.1"
 serial_test = "2.0.0"
 duplicate = "0.4.1"
 tempfile = "3.3.0"
-libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "435107c245112397914935c0f7148a18b91cafc6", features = [
+libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "774f830c7ffa62122a2cf8f3eae8c64a3ca5a8d7", features = [
     "test-utils",
 ] }

crates/datadog-trace-agent/src/config.rs

@@ -84,6 +84,8 @@ pub struct Config {
     pub stats_flush_interval: u64,
     /// how often to flush traces, in seconds
     pub trace_flush_interval: u64,
+    /// buffer length for span concentrator
+    pub stats_buffer_length: usize,
     pub trace_intake: Endpoint,
     pub trace_stats_intake: Endpoint,
     /// timeout for environment verification, in milliseconds
@@ -131,6 +133,11 @@ impl Config {
             Tags::new()
         };
 
+        let stats_buffer_length: usize = env::var("DD_STATS_BUFFER_LENGTH")
+            .ok()
+            .and_then(|val| val.parse::<usize>().ok())
+            .unwrap_or(90);
+
         #[allow(clippy::unwrap_used)]
         Ok(Config {
             app_name: Some(app_name),
@@ -157,6 +164,7 @@ impl Config {
                 .or_else(|_| env::var("HTTPS_PROXY"))
                 .ok(),
             tags,
+            stats_buffer_length,
         })
     }
 }

crates/datadog-trace-agent/src/lib.rs

@@ -12,7 +12,9 @@ pub mod config;
 pub mod env_verifier;
 pub mod http_utils;
 pub mod mini_agent;
+pub mod stats_concentrator_service;
 pub mod stats_flusher;
+pub mod stats_generator;
 pub mod stats_processor;
 pub mod trace_flusher;
 pub mod trace_processor;