From 745b234f61fbaa86d987bf1375302752548fed67 Mon Sep 17 00:00:00 2001 From: Tianning Li Date: Fri, 27 Mar 2026 09:52:00 -0400 Subject: [PATCH 01/10] feat(lambda-lite): detect Lambda Lite environment and write mini agent sentinel file (#81) * Add .worktrees to .gitignore Preparing for git worktree usage to enable isolated development workspaces. Co-Authored-By: Claude Sonnet 4.5 * feat(lambda-lite): detect Lambda Lite and write mini agent sentinel file - Add `is_lambda_lite()` in http_utils to detect Lambda Lite via `AWS_LAMBDA_INITIALIZATION_TYPE=native-http`; includes unit tests for all env var states (native-http, on-demand, empty, unset) - Write `/tmp/datadog/mini_agent_ready` sentinel on startup when running in Lambda Lite mode so dd-trace Node.js can switch from LogExporter (stdout) to AgentExporter (HTTP :8126) - Refine release profile: use fat LTO, explicit symbol stripping, and `panic = "abort"` for smaller binary size In standard Lambda, dd-trace detects a running agent via the Extension path `/opt/extensions/datadog-agent`. Lambda Lite (web function / native-http mode) does not populate this path, and `/opt` is read-only, so the standard detection mechanism does not apply. Without an agent signal, dd-trace falls back to LogExporter and writes traces to stdout where they are silently dropped. The sentinel file at `/tmp/datadog/mini_agent_ready` is written after the mini agent binds to :8126. dd-trace (Node.js) checks this path via `DATADOG_MINI_AGENT_PATH` in constants.js and switches to AgentExporter (HTTP :8126) when the file is present. `/tmp` is used because it is the only writable directory in Lambda Lite; the parent directory `/tmp/datadog/` is created by the serverless-compat JS layer before this binary is spawned. --------- Co-authored-by: Claude Sonnet 4.5 --- .gitignore | 1 + crates/datadog-trace-agent/src/http_utils.rs | 40 ++++++++++++++++++++ crates/datadog-trace-agent/src/mini_agent.rs | 39 ++++++++++++++++++- 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index df769d2..0dd34a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target /.idea +/.worktrees diff --git a/crates/datadog-trace-agent/src/http_utils.rs b/crates/datadog-trace-agent/src/http_utils.rs index 74bc510..a15019f 100644 --- a/crates/datadog-trace-agent/src/http_utils.rs +++ b/crates/datadog-trace-agent/src/http_utils.rs @@ -113,6 +113,25 @@ pub fn verify_request_content_length( None } +/// Environment variable set by the Lambda runtime to indicate the initialisation type. +/// Lambda Web Adapter (web function / native-http mode) sets this to `"native-http"`; +/// standard on-demand invocations set it to `"on-demand"`. +const ENV_LAMBDA_INIT_TYPE: &str = "AWS_LAMBDA_INITIALIZATION_TYPE"; + +/// Returns true if the current environment is Lambda Lite (web function / native-http mode). +/// +/// Determined by checking [`ENV_LAMBDA_INIT_TYPE`] == `"native-http"`. This is used to gate +/// behaviour specific to long-running web server deployments on Lambda Lite. +pub fn is_lambda_lite() -> bool { + is_lambda_lite_from_env(std::env::var(ENV_LAMBDA_INIT_TYPE).ok().as_deref()) +} + +// Split out from `is_lambda_lite` for testability — allows injecting the env value in unit tests +// without mutating process-global state. +fn is_lambda_lite_from_env(val: Option<&str>) -> bool { + val == Some("native-http") +} + /// Builds a reqwest client with optional proxy configuration and timeout. /// Uses rustls TLS by default. FIPS-compliant TLS is available via the fips feature pub fn build_client( @@ -134,8 +153,29 @@ mod tests { use hyper::header; use libdd_common::hyper_migration; + use super::is_lambda_lite_from_env; use super::verify_request_content_length; + #[test] + fn test_is_lambda_lite_native_http() { + assert!(is_lambda_lite_from_env(Some("native-http"))); + } + + #[test] + fn test_is_lambda_lite_on_demand() { + assert!(!is_lambda_lite_from_env(Some("on-demand"))); + } + + #[test] + fn test_is_lambda_lite_empty_string() { + assert!(!is_lambda_lite_from_env(Some(""))); + } + + #[test] + fn test_is_lambda_lite_unset() { + assert!(!is_lambda_lite_from_env(None)); + } + fn create_test_headers_with_content_length(val: &str) -> HeaderMap { let mut map = HeaderMap::new(); map.insert(header::CONTENT_LENGTH, val.parse().unwrap()); diff --git a/crates/datadog-trace-agent/src/mini_agent.rs b/crates/datadog-trace-agent/src/mini_agent.rs index 855290c..0d19b38 100644 --- a/crates/datadog-trace-agent/src/mini_agent.rs +++ b/crates/datadog-trace-agent/src/mini_agent.rs @@ -11,7 +11,7 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Instant; use tokio::sync::mpsc::{self, Receiver, Sender}; -use tracing::{debug, error}; +use tracing::{debug, error, warn}; use crate::http_utils::{log_and_create_http_response, verify_request_content_length}; use crate::proxy_flusher::{ProxyFlusher, ProxyRequest}; @@ -31,6 +31,13 @@ const PROFILING_ENDPOINT_PATH: &str = "/profiling/v1/input"; const TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; const STATS_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; const PROXY_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; +/// Sentinel file written on startup in Lambda Lite mode. +/// dd-trace (Node.js) checks this path via DATADOG_MINI_AGENT_PATH in constants.js +/// (datadog/dd-trace-js) to decide whether to switch from LogExporter (stdout) to +/// AgentExporter (HTTP :8126). +/// The parent directory `/tmp/datadog/` is created by the serverless-compat JS layer +/// before this binary is spawned. +const LAMBDA_LITE_SENTINEL_PATH: &str = "/tmp/datadog/mini_agent_ready"; pub struct MiniAgent { pub config: Arc, @@ -173,6 +180,36 @@ impl MiniAgent { let addr = SocketAddr::from(([127, 0, 0, 1], self.config.dd_apm_receiver_port)); let listener = tokio::net::TcpListener::bind(&addr).await?; + // Write the sentinel file after the listener is bound so that dd-trace + // (Node.js) only switches from LogExporter (stdout) to AgentExporter + // (HTTP :8126) once the port is actually ready to accept connections. + // Only written for Lambda Lite; standard Lambda invocations use the + // Extension path (/opt/extensions/datadog-agent) instead. + // /opt is read-only in Lambda Lite, so we use /tmp/datadog/ (created + // by the serverless-compat JS layer before spawning this binary). + if crate::http_utils::is_lambda_lite() { + let sentinel = std::path::Path::new(LAMBDA_LITE_SENTINEL_PATH); + // SAFETY: LAMBDA_LITE_SENTINEL_PATH is a hard-coded absolute path, + // so .parent() always returns Some. + if let Some(parent) = sentinel.parent() { + if let Err(e) = tokio::fs::create_dir_all(parent).await { + error!( + "Could not create parent directory for Lambda Lite sentinel \ + file at {}: {}.", + LAMBDA_LITE_SENTINEL_PATH, e + ); + } + } + if let Err(e) = tokio::fs::write(sentinel, b"").await { + error!( + "Could not write Lambda Lite sentinel file at {}: {}. \ + dd-trace (Node.js) will fall back to LogExporter (stdout), \ + traces may not reach Datadog.", + LAMBDA_LITE_SENTINEL_PATH, e + ); + } + } + Self::serve_tcp( listener, service, From 05e5c26a009f359a12501405b5e0f1d2641b1880 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:15:35 -0400 Subject: [PATCH 02/10] chore: update libdatadog rev to 8c88979985154d6d97c0fc2ca9039682981eacad (#107) * update libdatadog rev to 8c88979985154d6d97c0fc2ca9039682981eacad * update licenses --- Cargo.lock | 146 +++++++++++------- LICENSE-3rdparty.csv | 4 + crates/datadog-agent-config/Cargo.toml | 4 +- crates/datadog-serverless-compat/Cargo.toml | 2 +- crates/datadog-trace-agent/Cargo.toml | 10 +- .../datadog-trace-agent/src/env_verifier.rs | 34 ++-- crates/datadog-trace-agent/src/http_utils.rs | 16 +- crates/datadog-trace-agent/src/mini_agent.rs | 20 +-- .../src/stats_processor.rs | 10 +- .../datadog-trace-agent/src/trace_flusher.rs | 6 +- .../src/trace_processor.rs | 16 +- .../tests/common/helpers.rs | 10 +- .../tests/common/mock_server.rs | 4 +- .../datadog-trace-agent/tests/common/mocks.rs | 14 +- 14 files changed, 170 insertions(+), 126 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a6e243..9a7c7be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,6 +212,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + [[package]] name = "bumpalo" version = "3.20.2" @@ -442,7 +448,7 @@ dependencies = [ "dogstatsd", "figment", "libdd-trace-obfuscation", - "libdd-trace-utils 1.0.0", + "libdd-trace-utils 3.0.0", "log", "serde", "serde-aux", @@ -515,7 +521,7 @@ dependencies = [ "datadog-fips", "datadog-trace-agent", "dogstatsd", - "libdd-trace-utils 1.0.0", + "libdd-trace-utils 3.0.0", "reqwest", "tokio", "tokio-util", @@ -537,10 +543,10 @@ dependencies = [ "hyper", "hyper-http-proxy", "hyper-util", - "libdd-common 1.1.0", + "libdd-common 3.0.1", "libdd-trace-obfuscation", - "libdd-trace-protobuf 1.0.0", - "libdd-trace-utils 1.0.0", + "libdd-trace-protobuf 3.0.0", + "libdd-trace-utils 3.0.0", "reqwest", "rmp-serde", "serde", @@ -744,6 +750,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1394,10 +1411,12 @@ checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "libdd-common" -version = "1.1.0" -source = "git+https://github.com/DataDog/libdatadog?rev=d52ee90209cb12a28bdda0114535c1a985a29d95#d52ee90209cb12a28bdda0114535c1a985a29d95" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e5593b91f61eee38cddc9fdcbc99c9fad697b5d925e226bd500d86b4295380b" dependencies = [ "anyhow", + "bytes", "cc", "const_format", "futures", @@ -1408,28 +1427,23 @@ dependencies = [ "http-body", "http-body-util", "hyper", - "hyper-rustls", "hyper-util", "libc", "nix", "pin-project", "regex", - "rustls", - "rustls-native-certs", "serde", "static_assertions", "thiserror 1.0.69", "tokio", - "tokio-rustls", "tower-service", "windows-sys 0.52.0", ] [[package]] name = "libdd-common" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e5593b91f61eee38cddc9fdcbc99c9fad697b5d925e226bd500d86b4295380b" +version = "3.0.1" +source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" dependencies = [ "anyhow", "bytes", @@ -1443,15 +1457,19 @@ dependencies = [ "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-util", "libc", "nix", "pin-project", "regex", + "rustls", + "rustls-native-certs", "serde", "static_assertions", "thiserror 1.0.69", "tokio", + "tokio-rustls", "tower-service", "windows-sys 0.52.0", ] @@ -1546,51 +1564,52 @@ dependencies = [ [[package]] name = "libdd-tinybytes" version = "1.1.0" -source = "git+https://github.com/DataDog/libdatadog?rev=d52ee90209cb12a28bdda0114535c1a985a29d95#d52ee90209cb12a28bdda0114535c1a985a29d95" +source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" dependencies = [ "serde", ] [[package]] name = "libdd-trace-normalization" -version = "1.0.0" -source = "git+https://github.com/DataDog/libdatadog?rev=d52ee90209cb12a28bdda0114535c1a985a29d95#d52ee90209cb12a28bdda0114535c1a985a29d95" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a737b43f01d6a0cbd1399c5b89863a5d2663fe7b19bf1d3ea28048abab396353" dependencies = [ "anyhow", - "libdd-trace-protobuf 1.0.0", + "libdd-trace-protobuf 2.0.0", ] [[package]] name = "libdd-trace-normalization" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a737b43f01d6a0cbd1399c5b89863a5d2663fe7b19bf1d3ea28048abab396353" +version = "1.0.3" +source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" dependencies = [ "anyhow", - "libdd-trace-protobuf 2.0.0", + "libdd-trace-protobuf 3.0.0", ] [[package]] name = "libdd-trace-obfuscation" -version = "1.0.0" -source = "git+https://github.com/DataDog/libdatadog?rev=d52ee90209cb12a28bdda0114535c1a985a29d95#d52ee90209cb12a28bdda0114535c1a985a29d95" +version = "1.0.1" +source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" dependencies = [ "anyhow", - "libdd-common 1.1.0", - "libdd-trace-protobuf 1.0.0", - "libdd-trace-utils 1.0.0", + "fluent-uri", + "libdd-common 3.0.1", + "libdd-trace-protobuf 3.0.0", + "libdd-trace-utils 3.0.0", "log", "percent-encoding", "regex", "serde", "serde_json", - "url", ] [[package]] name = "libdd-trace-protobuf" -version = "1.0.0" -source = "git+https://github.com/DataDog/libdatadog?rev=d52ee90209cb12a28bdda0114535c1a985a29d95#d52ee90209cb12a28bdda0114535c1a985a29d95" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0a54921e03174f3ff7ad8506ff9e13637e546ef0b1f369ae463eacebda8e88" dependencies = [ "prost 0.14.3", "serde", @@ -1599,9 +1618,8 @@ dependencies = [ [[package]] name = "libdd-trace-protobuf" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0a54921e03174f3ff7ad8506ff9e13637e546ef0b1f369ae463eacebda8e88" +version = "3.0.0" +source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" dependencies = [ "prost 0.14.3", "serde", @@ -1622,24 +1640,21 @@ dependencies = [ [[package]] name = "libdd-trace-utils" -version = "1.0.0" -source = "git+https://github.com/DataDog/libdatadog?rev=d52ee90209cb12a28bdda0114535c1a985a29d95#d52ee90209cb12a28bdda0114535c1a985a29d95" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a59e9a0a41bb17d06fb85a70db3be04e53ddfb8f61a593939bb9677729214db" dependencies = [ "anyhow", "bytes", - "cargo-platform", - "cargo_metadata", - "flate2", "futures", "http", + "http-body", "http-body-util", - "httpmock", - "hyper", "indexmap", - "libdd-common 1.1.0", - "libdd-tinybytes 1.1.0 (git+https://github.com/DataDog/libdatadog?rev=d52ee90209cb12a28bdda0114535c1a985a29d95)", - "libdd-trace-normalization 1.0.0", - "libdd-trace-protobuf 1.0.0", + "libdd-common 2.0.1", + "libdd-tinybytes 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libdd-trace-normalization 1.0.2", + "libdd-trace-protobuf 2.0.0", "prost 0.14.3", "rand 0.8.5", "rmp", @@ -1649,27 +1664,29 @@ dependencies = [ "serde_json", "tokio", "tracing", - "urlencoding", - "zstd", ] [[package]] name = "libdd-trace-utils" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a59e9a0a41bb17d06fb85a70db3be04e53ddfb8f61a593939bb9677729214db" +version = "3.0.0" +source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" dependencies = [ "anyhow", "bytes", + "cargo-platform", + "cargo_metadata", + "flate2", "futures", "http", "http-body", "http-body-util", + "httpmock", + "hyper", "indexmap", - "libdd-common 2.0.1", - "libdd-tinybytes 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libdd-trace-normalization 1.0.2", - "libdd-trace-protobuf 2.0.0", + "libdd-common 3.0.1", + "libdd-tinybytes 1.1.0 (git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad)", + "libdd-trace-normalization 1.0.3", + "libdd-trace-protobuf 3.0.0", "prost 0.14.3", "rand 0.8.5", "rmp", @@ -1679,6 +1696,8 @@ dependencies = [ "serde_json", "tokio", "tracing", + "urlencoding", + "zstd", ] [[package]] @@ -2415,6 +2434,26 @@ dependencies = [ "bitflags", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "regex" version = "1.12.3" @@ -2764,6 +2803,7 @@ version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ + "indexmap", "itoa", "memchr", "serde", diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 012085e..2d04b2a 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -20,6 +20,7 @@ bit-set,https://github.com/contain-rs/bit-set,Apache-2.0 OR MIT,Alexis Beingessn bit-vec,https://github.com/contain-rs/bit-vec,Apache-2.0 OR MIT,Alexis Beingessner bitflags,https://github.com/bitflags/bitflags,MIT OR Apache-2.0,The Rust Project Developers block-buffer,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers +borrow-or-share,https://github.com/yescallop/borrow-or-share,MIT-0,Scallop Ye bumpalo,https://github.com/fitzgen/bumpalo,MIT OR Apache-2.0,Nick Fitzgerald bytemuck,https://github.com/Lokathor/bytemuck,Zlib OR Apache-2.0 OR MIT,Lokathor byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallant @@ -61,6 +62,7 @@ find-msvc-tools,https://github.com/rust-lang/cc-rs,MIT OR Apache-2.0,The find-ms fixedbitset,https://github.com/petgraph/fixedbitset,MIT OR Apache-2.0,bluss flate2,https://github.com/rust-lang/flate2-rs,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett " float-cmp,https://github.com/mikedilger/float-cmp,MIT,Mike Dilger +fluent-uri,https://github.com/yescallop/fluent-uri-rs,MIT,Scallop Ye fnv,https://github.com/servo/rust-fnv,Apache-2.0 OR MIT,Alex Crichton foldhash,https://github.com/orlp/foldhash,Zlib,Orson Peters form_urlencoded,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers @@ -188,6 +190,8 @@ rand_chacha,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Proj rand_core,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" rand_xorshift,https://github.com/rust-random/rngs,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" redox_syscall,https://gitlab.redox-os.org/redox-os/syscall,MIT,Jeremy Soller +ref-cast,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay +ref-cast-impl,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay regex,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-automata,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-syntax,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " diff --git a/crates/datadog-agent-config/Cargo.toml b/crates/datadog-agent-config/Cargo.toml index 089537b..222d726 100644 --- a/crates/datadog-agent-config/Cargo.toml +++ b/crates/datadog-agent-config/Cargo.toml @@ -9,8 +9,8 @@ path = "mod.rs" [dependencies] figment = { version = "0.10", default-features = false, features = ["yaml", "env"] } -libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95" } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95" } +libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } log = { version = "0.4", default-features = false } serde = { version = "1.0", default-features = false, features = ["derive"] } serde-aux = { version = "4.7", default-features = false } diff --git a/crates/datadog-serverless-compat/Cargo.toml b/crates/datadog-serverless-compat/Cargo.toml index ed57366..b41de18 100644 --- a/crates/datadog-serverless-compat/Cargo.toml +++ b/crates/datadog-serverless-compat/Cargo.toml @@ -11,7 +11,7 @@ windows-pipes = ["datadog-trace-agent/windows-pipes", "dogstatsd/windows-pipes"] [dependencies] datadog-trace-agent = { path = "../datadog-trace-agent" } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95" } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } datadog-fips = { path = "../datadog-fips", default-features = false } dogstatsd = { path = "../dogstatsd", default-features = true } reqwest = { version = "0.12.4", default-features = false } diff --git a/crates/datadog-trace-agent/Cargo.toml b/crates/datadog-trace-agent/Cargo.toml index 4cc5ed7..1a69733 100644 --- a/crates/datadog-trace-agent/Cargo.toml +++ b/crates/datadog-trace-agent/Cargo.toml @@ -24,12 +24,12 @@ async-trait = "0.1.64" tracing = { version = "0.1", default-features = false } serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0" -libdd-common = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95" } -libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95" } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95", features = [ +libdd-common = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } +libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad", features = [ "mini_agent", ] } -libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95" } +libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } datadog-fips = { path = "../datadog-fips" } reqwest = { version = "0.12.23", features = ["json", "http2"], default-features = false } bytes = "1.10.1" @@ -40,6 +40,6 @@ serial_test = "2.0.0" duplicate = "0.4.1" temp-env = "0.3.6" tempfile = "3.3.0" -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95", features = [ +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad", features = [ "test-utils", ] } diff --git a/crates/datadog-trace-agent/src/env_verifier.rs b/crates/datadog-trace-agent/src/env_verifier.rs index 29fcc5c..9ece700 100644 --- a/crates/datadog-trace-agent/src/env_verifier.rs +++ b/crates/datadog-trace-agent/src/env_verifier.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use http_body_util::BodyExt; use hyper::{Method, Request}; -use libdd_common::hyper_migration; +use libdd_common::http_common; use serde::{Deserialize, Serialize}; use std::env; use std::fs; @@ -186,24 +186,24 @@ fn get_region_from_gcp_region_string(str: String) -> String { /// tests #[async_trait] pub(crate) trait GoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result; + async fn get_metadata(&self) -> anyhow::Result; } struct GoogleMetadataClientWrapper {} #[async_trait] impl GoogleMetadataClient for GoogleMetadataClientWrapper { - async fn get_metadata(&self) -> anyhow::Result { + async fn get_metadata(&self) -> anyhow::Result { let req = Request::builder() .method(Method::POST) .uri(GCP_METADATA_URL) .header("Metadata-Flavor", "Google") - .body(hyper_migration::Body::empty()) + .body(http_common::Body::empty()) .map_err(|err| anyhow::anyhow!(err.to_string()))?; - let client = hyper_migration::new_default_client(); + let client = http_common::new_default_client(); match client.request(req).await { - Ok(res) => Ok(hyper_migration::into_response(res)), + Ok(res) => Ok(http_common::into_response(res)), Err(err) => anyhow::bail!(err.to_string()), } } @@ -243,7 +243,7 @@ async fn ensure_gcp_function_environment( Ok(gcp_metadata) } -async fn get_gcp_metadata_from_body(body: hyper_migration::Body) -> anyhow::Result { +async fn get_gcp_metadata_from_body(body: http_common::Body) -> anyhow::Result { let bytes = body.collect().await?.to_bytes(); let body_str = String::from_utf8(bytes.to_vec())?; let gcp_metadata: GCPMetadata = serde_json::from_str(&body_str)?; @@ -361,7 +361,7 @@ async fn ensure_azure_function_environment( mod tests { use async_trait::async_trait; use hyper::{Response, StatusCode, body::Bytes}; - use libdd_common::hyper_migration; + use libdd_common::http_common; use libdd_trace_utils::trace_utils; use serde_json::json; use serial_test::serial; @@ -382,7 +382,7 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result { + async fn get_metadata(&self) -> anyhow::Result { anyhow::bail!("Random Error") } } @@ -401,9 +401,9 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result { + async fn get_metadata(&self) -> anyhow::Result { Ok( - hyper_migration::empty_response(Response::builder().status(StatusCode::OK)) + http_common::empty_response(Response::builder().status(StatusCode::OK)) .unwrap(), ) } @@ -423,8 +423,8 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result { - Ok(hyper_migration::empty_response( + async fn get_metadata(&self) -> anyhow::Result { + Ok(http_common::empty_response( Response::builder() .status(StatusCode::OK) .header("Server", "Metadata Server NOT for Serverless"), @@ -447,8 +447,8 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result { - Ok(hyper_migration::mock_response( + async fn get_metadata(&self) -> anyhow::Result { + Ok(http_common::mock_response( Response::builder() .status(StatusCode::OK) .header("Server", "Metadata Server for Serverless"), @@ -489,11 +489,11 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result { + async fn get_metadata(&self) -> anyhow::Result { // Sleep for 5 seconds to let the timeout trigger tokio::time::sleep(Duration::from_secs(5)).await; Ok( - hyper_migration::empty_response(Response::builder().status(StatusCode::OK)) + http_common::empty_response(Response::builder().status(StatusCode::OK)) .unwrap(), ) } diff --git a/crates/datadog-trace-agent/src/http_utils.rs b/crates/datadog-trace-agent/src/http_utils.rs index a15019f..6061566 100644 --- a/crates/datadog-trace-agent/src/http_utils.rs +++ b/crates/datadog-trace-agent/src/http_utils.rs @@ -7,7 +7,7 @@ use hyper::{ Response, StatusCode, header, http::{self, HeaderMap}, }; -use libdd_common::hyper_migration; +use libdd_common::http_common; use serde_json::json; use std::error::Error; use tracing::{debug, error}; @@ -24,7 +24,7 @@ use tracing::{debug, error}; pub fn log_and_create_http_response( message: &str, status: StatusCode, -) -> http::Result> { +) -> http::Result> { if status.is_success() { debug!("{message}"); } else { @@ -33,7 +33,7 @@ pub fn log_and_create_http_response( let body = json!({ "message": message }).to_string(); Response::builder() .status(status) - .body(hyper_migration::Body::from(body)) + .body(http_common::Body::from(body)) } /// Does two things: @@ -50,12 +50,12 @@ pub fn log_and_create_http_response( pub fn log_and_create_traces_success_http_response( message: &str, status: StatusCode, -) -> http::Result { +) -> http::Result { debug!("{message}"); let body = json!({"rate_by_service":{"service:,env:":1}}).to_string(); Response::builder() .status(status) - .body(hyper_migration::Body::from(body)) + .body(http_common::Body::from(body)) } /// Takes a request's header map, and verifies that the "content-length" and/or "Transfer-Encoding" header @@ -67,7 +67,7 @@ pub fn verify_request_content_length( header_map: &HeaderMap, max_content_length: usize, error_message_prefix: &str, -) -> Option> { +) -> Option> { let content_length_header = match header_map.get(header::CONTENT_LENGTH) { Some(res) => res, None => { @@ -151,7 +151,7 @@ mod tests { use hyper::HeaderMap; use hyper::StatusCode; use hyper::header; - use libdd_common::hyper_migration; + use libdd_common::http_common; use super::is_lambda_lite_from_env; use super::verify_request_content_length; @@ -182,7 +182,7 @@ mod tests { map } - async fn get_response_body_as_string(response: hyper_migration::HttpResponse) -> String { + async fn get_response_body_as_string(response: http_common::HttpResponse) -> String { let body = response.into_body(); let bytes = body.collect().await.unwrap().to_bytes(); String::from_utf8(bytes.into_iter().collect()).unwrap() diff --git a/crates/datadog-trace-agent/src/mini_agent.rs b/crates/datadog-trace-agent/src/mini_agent.rs index 0d19b38..7fa5a92 100644 --- a/crates/datadog-trace-agent/src/mini_agent.rs +++ b/crates/datadog-trace-agent/src/mini_agent.rs @@ -4,7 +4,7 @@ use http_body_util::BodyExt; use hyper::service::service_fn; use hyper::{Method, Response, StatusCode, http}; -use libdd_common::hyper_migration; +use libdd_common::http_common; use serde_json::json; use std::io; use std::net::SocketAddr; @@ -125,7 +125,7 @@ impl MiniAgent { MiniAgent::trace_endpoint_handler( endpoint_config, - req.map(hyper_migration::Body::incoming), + req.map(http_common::Body::incoming), trace_processor, trace_tx, stats_processor, @@ -231,7 +231,7 @@ impl MiniAgent { where S: hyper::service::Service< hyper::Request, - Response = hyper::Response, + Response = hyper::Response, > + Clone + Send + 'static, @@ -302,7 +302,7 @@ impl MiniAgent { where S: hyper::service::Service< hyper::Request, - Response = hyper::Response, + Response = hyper::Response, > + Clone + Send + 'static, @@ -385,14 +385,14 @@ impl MiniAgent { #[allow(clippy::too_many_arguments)] async fn trace_endpoint_handler( config: Arc, - req: hyper_migration::HttpRequest, + req: http_common::HttpRequest, trace_processor: Arc, trace_tx: Sender, stats_processor: Arc, stats_tx: Sender, mini_agent_metadata: Arc, proxy_tx: Sender, - ) -> http::Result { + ) -> http::Result { match (req.method(), req.uri().path()) { (&Method::PUT | &Method::POST, TRACE_ENDPOINT_PATH) => { match trace_processor @@ -449,9 +449,9 @@ impl MiniAgent { /// Handles incoming proxy requests for profiling - can be abstracted into a generic proxy handler for other proxy requests in the future async fn profiling_proxy_handler( config: Arc, - request: hyper_migration::HttpRequest, + request: http_common::HttpRequest, proxy_tx: Sender, - ) -> http::Result { + ) -> http::Result { debug!("Received profiling request"); // Extract headers and body @@ -503,7 +503,7 @@ impl MiniAgent { dd_apm_receiver_port: u16, dd_apm_windows_pipe_name: Option<&str>, dd_dogstatsd_port: u16, - ) -> http::Result { + ) -> http::Result { // pipe_name already includes \\.\pipe\ prefix from config let receiver_socket = dd_apm_windows_pipe_name.unwrap_or(""); @@ -527,6 +527,6 @@ impl MiniAgent { ); Response::builder() .status(200) - .body(hyper_migration::Body::from(response_json.to_string())) + .body(http_common::Body::from(response_json.to_string())) } } diff --git a/crates/datadog-trace-agent/src/stats_processor.rs b/crates/datadog-trace-agent/src/stats_processor.rs index 889e5f2..aa47a41 100644 --- a/crates/datadog-trace-agent/src/stats_processor.rs +++ b/crates/datadog-trace-agent/src/stats_processor.rs @@ -6,7 +6,7 @@ use std::time::UNIX_EPOCH; use async_trait::async_trait; use hyper::{StatusCode, http}; -use libdd_common::hyper_migration; +use libdd_common::http_common; use tokio::sync::mpsc::Sender; use tracing::debug; @@ -23,9 +23,9 @@ pub trait StatsProcessor { async fn process_stats( &self, config: Arc, - req: hyper_migration::HttpRequest, + req: http_common::HttpRequest, tx: Sender, - ) -> http::Result; + ) -> http::Result; } #[derive(Clone)] @@ -36,9 +36,9 @@ impl StatsProcessor for ServerlessStatsProcessor { async fn process_stats( &self, config: Arc, - req: hyper_migration::HttpRequest, + req: http_common::HttpRequest, tx: Sender, - ) -> http::Result { + ) -> http::Result { debug!("Received trace stats to process"); let (parts, body) = req.into_parts(); diff --git a/crates/datadog-trace-agent/src/trace_flusher.rs b/crates/datadog-trace-agent/src/trace_flusher.rs index cf2619e..9efebac 100644 --- a/crates/datadog-trace-agent/src/trace_flusher.rs +++ b/crates/datadog-trace-agent/src/trace_flusher.rs @@ -6,7 +6,7 @@ use std::{error::Error, sync::Arc, time}; use tokio::sync::{Mutex, mpsc::Receiver}; use tracing::{debug, error}; -use libdd_common::{GenericHttpClient, hyper_migration}; +use libdd_common::{GenericHttpClient, http_common}; use libdd_trace_utils::trace_utils; use libdd_trace_utils::trace_utils::SendData; @@ -111,12 +111,12 @@ impl ServerlessTraceFlusher { libdd_common::connector::Connector::default(), proxy, )?; - Ok(hyper_migration::client_builder().build(proxy_connector)) + Ok(http_common::client_builder().build(proxy_connector)) } else { let proxy_connector = hyper_http_proxy::ProxyConnector::new( libdd_common::connector::Connector::default(), )?; - Ok(hyper_migration::client_builder().build(proxy_connector)) + Ok(http_common::client_builder().build(proxy_connector)) } } } diff --git a/crates/datadog-trace-agent/src/trace_processor.rs b/crates/datadog-trace-agent/src/trace_processor.rs index 1685137..96f8209 100644 --- a/crates/datadog-trace-agent/src/trace_processor.rs +++ b/crates/datadog-trace-agent/src/trace_processor.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use async_trait::async_trait; use hyper::{StatusCode, http}; -use libdd_common::hyper_migration; +use libdd_common::http_common; use tokio::sync::mpsc::Sender; use tracing::debug; @@ -29,10 +29,10 @@ pub trait TraceProcessor { async fn process_traces( &self, config: Arc, - req: hyper_migration::HttpRequest, + req: http_common::HttpRequest, tx: Sender, mini_agent_metadata: Arc, - ) -> http::Result; + ) -> http::Result; } struct ChunkProcessor { @@ -72,10 +72,10 @@ impl TraceProcessor for ServerlessTraceProcessor { async fn process_traces( &self, config: Arc, - req: hyper_migration::HttpRequest, + req: http_common::HttpRequest, tx: Sender, mini_agent_metadata: Arc, - ) -> http::Result { + ) -> http::Result { debug!("Received traces to process"); let (parts, body) = req.into_parts(); @@ -170,7 +170,7 @@ mod tests { config::{Config, Tags}, trace_processor::{self, TRACER_PAYLOAD_FUNCTION_TAGS_TAG_KEY, TraceProcessor}, }; - use libdd_common::{Endpoint, hyper_migration}; + use libdd_common::{Endpoint, http_common}; use libdd_trace_protobuf::pb; use libdd_trace_utils::test_utils::{create_test_gcp_json_span, create_test_gcp_span}; use libdd_trace_utils::trace_utils::MiniAgentMetadata; @@ -251,7 +251,7 @@ mod tests { .header("datadog-meta-lang-interpreter", "v8") .header("datadog-container-id", "33") .header("content-length", "100") - .body(hyper_migration::Body::from(bytes)) + .body(http_common::Body::from(bytes)) .unwrap(); let trace_processor = trace_processor::ServerlessTraceProcessor {}; @@ -323,7 +323,7 @@ mod tests { .header("datadog-meta-lang-interpreter", "v8") .header("datadog-container-id", "33") .header("content-length", "100") - .body(hyper_migration::Body::from(bytes)) + .body(http_common::Body::from(bytes)) .unwrap(); let trace_processor = trace_processor::ServerlessTraceProcessor {}; diff --git a/crates/datadog-trace-agent/tests/common/helpers.rs b/crates/datadog-trace-agent/tests/common/helpers.rs index 6f6e776..6dd8d82 100644 --- a/crates/datadog-trace-agent/tests/common/helpers.rs +++ b/crates/datadog-trace-agent/tests/common/helpers.rs @@ -5,7 +5,7 @@ use hyper::{Request, Response}; use hyper_util::rt::TokioIo; -use libdd_common::hyper_migration; +use libdd_common::http_common; use libdd_trace_utils::test_utils::create_test_json_span; use std::time::{Duration, UNIX_EPOCH}; use tokio::time::timeout; @@ -45,10 +45,10 @@ pub async fn send_tcp_request( let response = if let Some(body_data) = body { let body_len = body_data.len(); request_builder = request_builder.header("Content-Length", body_len.to_string()); - let request = request_builder.body(hyper_migration::Body::from(body_data))?; + let request = request_builder.body(http_common::Body::from(body_data))?; timeout(Duration::from_secs(2), sender.send_request(request)).await?? } else { - let request = request_builder.body(hyper_migration::Body::empty())?; + let request = request_builder.body(http_common::Body::empty())?; timeout(Duration::from_secs(2), sender.send_request(request)).await?? }; @@ -83,10 +83,10 @@ pub async fn send_named_pipe_request( let response = if let Some(body_data) = body { let body_len = body_data.len(); request_builder = request_builder.header("Content-Length", body_len.to_string()); - let request = request_builder.body(hyper_migration::Body::from(body_data))?; + let request = request_builder.body(http_common::Body::from(body_data))?; timeout(Duration::from_secs(2), sender.send_request(request)).await?? } else { - let request = request_builder.body(hyper_migration::Body::empty())?; + let request = request_builder.body(http_common::Body::empty())?; timeout(Duration::from_secs(2), sender.send_request(request)).await?? }; diff --git a/crates/datadog-trace-agent/tests/common/mock_server.rs b/crates/datadog-trace-agent/tests/common/mock_server.rs index b78b96c..f1beb1a 100644 --- a/crates/datadog-trace-agent/tests/common/mock_server.rs +++ b/crates/datadog-trace-agent/tests/common/mock_server.rs @@ -6,7 +6,7 @@ use http_body_util::BodyExt; use hyper::{Request, Response, body::Incoming}; use hyper_util::rt::TokioIo; -use libdd_common::hyper_migration; +use libdd_common::http_common; use std::net::SocketAddr; use std::sync::{Arc, Mutex}; use tokio::net::TcpListener; @@ -86,7 +86,7 @@ impl MockServer { Ok::<_, hyper::http::Error>( Response::builder() .status(200) - .body(hyper_migration::Body::from(r#"{"ok":true}"#)) + .body(http_common::Body::from(r#"{"ok":true}"#)) .unwrap(), ) } diff --git a/crates/datadog-trace-agent/tests/common/mocks.rs b/crates/datadog-trace-agent/tests/common/mocks.rs index d57aa2a..842c45f 100644 --- a/crates/datadog-trace-agent/tests/common/mocks.rs +++ b/crates/datadog-trace-agent/tests/common/mocks.rs @@ -7,7 +7,7 @@ use datadog_trace_agent::{ config::Config, env_verifier::EnvVerifier, stats_flusher::StatsFlusher, stats_processor::StatsProcessor, trace_flusher::TraceFlusher, trace_processor::TraceProcessor, }; -use libdd_common::hyper_migration; +use libdd_common::http_common; use libdd_trace_protobuf::pb; use libdd_trace_utils::trace_utils::{self, MiniAgentMetadata, SendData}; use std::sync::Arc; @@ -22,13 +22,13 @@ impl TraceProcessor for MockTraceProcessor { async fn process_traces( &self, _config: Arc, - _req: hyper_migration::HttpRequest, + _req: http_common::HttpRequest, _trace_tx: Sender, _mini_agent_metadata: Arc, - ) -> Result { + ) -> Result { hyper::Response::builder() .status(200) - .body(hyper_migration::Body::from("{}")) + .body(http_common::Body::from("{}")) } } @@ -68,12 +68,12 @@ impl StatsProcessor for MockStatsProcessor { async fn process_stats( &self, _config: Arc, - _req: hyper_migration::HttpRequest, + _req: http_common::HttpRequest, _stats_tx: Sender, - ) -> Result { + ) -> Result { hyper::Response::builder() .status(200) - .body(hyper_migration::Body::from("{}")) + .body(http_common::Body::from("{}")) } } From e3c16edeb10d0c281c46c780f1ffde06d8f7f08f Mon Sep 17 00:00:00 2001 From: Tianning Li Date: Tue, 31 Mar 2026 17:52:40 -0400 Subject: [PATCH 03/10] ci: add Copilot instructions for PII and security review (#109) Adds .github/copilot-instructions.md to guide GitHub Copilot auto-review toward security-relevant patterns on every PR: PII in log statements, unsafe Rust blocks without invariant documentation, and silently swallowed errors in network/external-input code paths. Jira: https://datadoghq.atlassian.net/browse/SVLS-8660 Co-authored-by: Claude Sonnet 4.6 --- .github/copilot-instructions.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..2091cfa --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,29 @@ +# Copilot Code Review Instructions + +## Security — PII and Secrets + +Flag any logging statements (`log::info!`, `log::debug!`, `log::warn!`, `log::error!`, +`tracing::info!`, `tracing::debug!`, `tracing::warn!`, `tracing::error!`, or unqualified +`info!`, `debug!`, `warn!`, `error!` macros (e.g., via `use tracing::{info, debug, warn, error}`)) +that may log: +- HTTP request/response headers (Authorization, Cookie, X-API-Key, or similar) +- HTTP request/response bodies or raw payloads +- Any PII fields (e.g., email, name, user_id, ip_address, phone, ssn, date_of_birth) +- API keys, tokens, secrets, or credentials +- Structs or types that contain any of the above fields +- `SendData` values or any variable that contains a `SendData` object (e.g., + `traces_with_tags` or similar variables built via `.with_api_key(...).build()`), + since these embed the Datadog API key + +Suggest redacting or omitting the sensitive field rather than logging it. + +## Security — Unsafe Rust + +Flag new `unsafe` blocks and explain what invariant the author must uphold to make the +block safe. If there is a safe alternative, suggest it. + +## Security — Error Handling + +Flag cases where errors are silently swallowed (empty `catch`, `.ok()` without +handling, `let _ = result`) or where operations like `.unwrap()`/`.expect()` may panic, +in code paths that handle external input or network responses. From bb2532aee6e9b57dc0fffdbf988e7a44bddbc084 Mon Sep 17 00:00:00 2001 From: Tianning Li Date: Thu, 2 Apr 2026 11:41:40 -0400 Subject: [PATCH 04/10] feat: add win32-ia32 and darwin-arm64 (macOS Apple Silicon) npm packages (#108) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add two new platform targets to the datadog-serverless-compat CI pipeline: - win32-ia32: 32-bit Windows build via native windows-2022 runner (i686-pc-windows-msvc, UPX-compressed) - darwin-arm64: macOS Apple Silicon build via native macos-14 runner (aarch64-apple-darwin, no UPX — preserves Mach-O code signing) Each platform adds a build step to build-datadog-serverless-compat.yml, artifact download/processing in the package job, and an npm publish line in the publish job of publish.yml. Co-authored-by: Claude Sonnet 4.6 --- .../build-datadog-serverless-compat.yml | 22 ++++++++++++++++ .github/workflows/publish.yml | 25 ++++++++++++++++++- .../package.json | 25 +++++++++++++++++++ .../package.json | 22 ++++++++++++++++ 4 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 npm/datadog-serverless-compat-darwin-arm64/package.json create mode 100644 npm/datadog-serverless-compat-win32-ia32/package.json diff --git a/.github/workflows/build-datadog-serverless-compat.yml b/.github/workflows/build-datadog-serverless-compat.yml index 0c72908..8c88500 100644 --- a/.github/workflows/build-datadog-serverless-compat.yml +++ b/.github/workflows/build-datadog-serverless-compat.yml @@ -63,3 +63,25 @@ jobs: name: windows-amd64 path: target/release/datadog-serverless-compat.exe retention-days: 3 + - if: ${{ inputs.runner == 'windows-2022' }} + shell: bash + run: | + rustup target add i686-pc-windows-msvc + cargo build --release -p datadog-serverless-compat \ + --target i686-pc-windows-msvc \ + --features windows-pipes + - if: ${{ inputs.runner == 'windows-2022' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 + with: + name: windows-ia32 + path: target/i686-pc-windows-msvc/release/datadog-serverless-compat.exe + retention-days: 3 + - if: ${{ inputs.runner == 'macos-14' }} + shell: bash + run: cargo build --release -p datadog-serverless-compat + - if: ${{ inputs.runner == 'macos-14' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 + with: + name: darwin-arm64 + path: target/release/datadog-serverless-compat + retention-days: 3 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1d37e4d..4dcc42b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - runner: [ubuntu-24.04, ubuntu-24.04-arm, windows-2022] + runner: [ubuntu-24.04, ubuntu-24.04-arm, windows-2022, macos-14] uses: ./.github/workflows/build-datadog-serverless-compat.yml with: runner: ${{ matrix.runner }} @@ -56,6 +56,16 @@ jobs: name: windows-amd64 path: target/windows-amd64 - run: upx target/windows-amd64/datadog-serverless-compat.exe --lzma + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # 4.3.0 + with: + name: windows-ia32 + path: target/windows-ia32 + - run: upx target/windows-ia32/datadog-serverless-compat.exe --lzma + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # 4.3.0 + with: + name: darwin-arm64 + path: target/darwin-arm64 + - run: chmod +x target/darwin-arm64/datadog-serverless-compat - name: Determine version id: determine-version env: @@ -85,6 +95,14 @@ jobs: mkdir -p npm/datadog-serverless-compat-win32-x64/bin cp target/windows-amd64/datadog-serverless-compat.exe npm/datadog-serverless-compat-win32-x64/bin/ npm --prefix npm/datadog-serverless-compat-win32-x64 pkg set version="$VERSION" + + mkdir -p npm/datadog-serverless-compat-win32-ia32/bin + cp target/windows-ia32/datadog-serverless-compat.exe npm/datadog-serverless-compat-win32-ia32/bin/ + npm --prefix npm/datadog-serverless-compat-win32-ia32 pkg set version="$VERSION" + + mkdir -p npm/datadog-serverless-compat-darwin-arm64/bin + cp target/darwin-arm64/datadog-serverless-compat npm/datadog-serverless-compat-darwin-arm64/bin/ + npm --prefix npm/datadog-serverless-compat-darwin-arm64 pkg set version="$VERSION" - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 with: name: npm-packages @@ -105,8 +123,13 @@ jobs: with: node-version: "22.x" registry-url: 'https://registry.npmjs.org' + - run: npm config set //registry.npmjs.org/:_authToken=$NPM_PUBLISH_TOKEN + env: + NPM_PUBLISH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} - name: Publish npm packages run: | npm publish ./npm/datadog-serverless-compat-linux-x64 --provenance --access public npm publish ./npm/datadog-serverless-compat-linux-arm64 --provenance --access public npm publish ./npm/datadog-serverless-compat-win32-x64 --provenance --access public + npm publish ./npm/datadog-serverless-compat-win32-ia32 --provenance --access public + npm publish ./npm/datadog-serverless-compat-darwin-arm64 --provenance --access public diff --git a/npm/datadog-serverless-compat-darwin-arm64/package.json b/npm/datadog-serverless-compat-darwin-arm64/package.json new file mode 100644 index 0000000..b4211e1 --- /dev/null +++ b/npm/datadog-serverless-compat-darwin-arm64/package.json @@ -0,0 +1,25 @@ +{ + "name": "@datadog/serverless-compat-darwin-arm64", + "version": "0.0.0", + "description": "macOS arm64 binary for the Datadog Serverless Compatibility Layer", + "os": [ + "darwin" + ], + "cpu": [ + "arm64" + ], + "files": [ + "bin/" + ], + "publishConfig": { + "access": "public", + "executableFiles": [ + "./bin/datadog-serverless-compat" + ] + }, + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/DataDog/serverless-components" + } +} diff --git a/npm/datadog-serverless-compat-win32-ia32/package.json b/npm/datadog-serverless-compat-win32-ia32/package.json new file mode 100644 index 0000000..7744c7b --- /dev/null +++ b/npm/datadog-serverless-compat-win32-ia32/package.json @@ -0,0 +1,22 @@ +{ + "name": "@datadog/serverless-compat-win32-ia32", + "version": "0.0.0", + "description": "Windows ia32 binary for the Datadog Serverless Compatibility Layer", + "os": [ + "win32" + ], + "cpu": [ + "ia32" + ], + "files": [ + "bin/" + ], + "publishConfig": { + "access": "public" + }, + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/DataDog/serverless-components" + } +} From 9148c8d22d2b83de1648ba9c3ab6203997f1742a Mon Sep 17 00:00:00 2001 From: Tianning Li Date: Mon, 6 Apr 2026 13:54:39 -0400 Subject: [PATCH 05/10] feat(logs-agent): add datadog-logs-agent crate and wire into serverless-compat (#95) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(log-agent): scaffold datadog-log-agent crate with constants and errors Co-Authored-By: Claude Sonnet 4.6 * - use serde flatten with Map instead of Option to avoid deser quirk - add generic LogEntry with flatten attributes for runtime enrichment - add features table and fix zstd compression level docs * feat(log-agent): add LogAggregator with size/count-bounded batch collection Co-Authored-By: Claude Sonnet 4.6 * refactor(log-agent): clean up get_batch loop to include comma bytes in tally Co-Authored-By: Claude Sonnet 4.6 * feat(log-agent): add AggregatorService + AggregatorHandle with channel pattern Co-Authored-By: Claude Sonnet 4.6 * feat(log-agent): add LogFlusher with zstd compression, retry logic, and OPW mode Co-Authored-By: Claude Sonnet 4.6 * fix(log-agent): align flusher tests with spec requirements Co-Authored-By: Claude Sonnet 4.6 * fix(log-agent): code quality improvements in flusher and config Co-Authored-By: Claude Sonnet 4.6 * feat(log-agent): integrate datadog-log-agent into serverless-compat binary - Clean public re-exports in datadog-log-agent lib.rs - Add datadog-log-agent dependency to datadog-serverless-compat - Wire log agent startup in main.rs following DogStatsD pattern - Respect DD_LOGS_ENABLED env var (default: true) - Use FIPS-compliant HTTP client via create_reqwest_client_builder - Flush logs on same interval as DogStatsD metrics - Add integration test verifying full pipeline compiles and runs - Update CLAUDE.md with log-agent architecture and env vars Co-Authored-By: Claude Sonnet 4.6 * fix(log-agent): improve main.rs wiring — early return on missing API key, use crate re-exports Co-Authored-By: Claude Sonnet 4.6 * test(log-agent): add integration test suite covering full pipeline, batching, retries, and OPW mode * feat(log-agent): derive Clone on LogFlusher * - add CLAUDE.md to .gitignore - disable log agent by default — require explicit DD_LOGS_ENABLED=true - log the actual error when reqwest client build() fails in start_log_agent - fail fast in start_log_agent when OPW URL is empty - apply rustfmt to integration test formatting. * chore(log-agent): add hyper/http-body-util/hyper-util deps for log server * feat(log-agent): add LogServer HTTP intake skeleton * test(log-agent): add LogServer HTTP intake integration tests * feat(serverless-compat): start LogServer HTTP intake on DD_LOGS_PORT (default 8080) * chore: remove stale TODO comment — LogServer wires the log-ingestion endpoint * test(log-agent): add unit and integration tests for LogServer network intake * test(log-agent): add network intake integration tests for LogServer Cover the full HTTP→LogServer→AggregatorService→LogFlusher→backend pipeline, concurrent client ingestion, and error recovery after a malformed request. Co-Authored-By: Claude Sonnet 4.6 * fix(log-agent): treat 408/425/429 as retryable instead of permanent 4xx All 4xx responses were previously short-circuited as permanent failures, causing rate-limited (429) and timed-out (408) batches to be silently dropped. These are transient conditions that should go through the existing retry loop. TODO: parse Retry-After header on 429 to add proper backoff. Co-Authored-By: Claude Sonnet 4.6 * fix(serverless-compat): warn when log agent flush fails Previously the boolean returned by LogFlusher::flush() was silently discarded, giving operators no signal when logs were being dropped. Now logs a warning on each failed flush cycle. TODO: expose as a statsd counter/gauge for durable telemetry. Co-Authored-By: Claude Sonnet 4.6 * fix(log-server): accept chunked requests by checking Content-Length header instead of size_hint size_hint().upper() returns None for chunked/streaming bodies; coercing that to u64::MAX caused every request without a Content-Length header to be rejected with 413 before any body bytes were read. Replace the pre-read guard with a direct Content-Length header parse: reject early only when the header is present and exceeds MAX_BODY_BYTES, and fall through to the post-read bytes.len() check otherwise. Adds a regression test that sends a raw Transfer-Encoding: chunked request (no Content-Length) via TcpStream and asserts 200 + correct aggregator insertion. * chore(log-agent): change default log intake port to 10517 and document bind-failure gap - Extract DEFAULT_LOG_INTAKE_PORT = 10517 constant (was hardcoded 8080) - Add TODO explaining that LogServer::serve binds inside the spawned task, so a port-conflict failure is silently swallowed while the caller still returns Some(...) and logs "log agent started" * feat(log-agent): send additional endpoints concurrently in flush() - Replace sequential for-loop over additional_endpoints with join_all() - Add futures crate dependency for join_all - Add unit tests: one verifying all endpoints receive the batch, one using Barrier(2) to prove concurrent in-flight dispatch Rationale: LogFlusherConfig documented additional_endpoints as shipped "in parallel" but the implementation was sequential — this aligns the implementation with the documented contract This commit made by [/dd:git:commit:quick](https://github.com/DataDog/claude-marketplace/tree/main/dd/commands/git/commit/quick.md) * perf(log-agent): dispatch all log batches concurrently in flush() - Replace sequential for-batch loop with join_all over all batches - Each batch now ships to primary and extras concurrently in parallel - Collect per-batch primary results via join_all then fold with .all() Rationale: multiple batches were flushed one at a time; concurrent dispatch reduces total flush latency when the aggregator produces more than one batch This commit made by [/dd:git:commit:quick](https://github.com/DataDog/claude-marketplace/tree/main/dd/commands/git/commit/quick.md) * fix(log-agent): drain response body after each HTTP send - Call resp.bytes().await after receiving a response to consume the body - Ensures the TCP connection is returned to the pool instead of lingering in CLOSE_WAIT, which would exhaust the connection pool under high flush frequency Rationale: reqwest reuses connections only after the response body is fully consumed; skipping this keeps connections open unnecessarily This commit made by [/dd:git:commit:quick](https://github.com/DataDog/claude-marketplace/tree/main/dd/commands/git/commit/quick.md) * feat(log-agent): per-endpoint API key for additional_endpoints - Add LogsAdditionalEndpoint {api_key, url, is_reliable} matching the bottlecap/datadog-agent wire format (Host+Port deserialized to url) - Add parse_additional_endpoints() and read DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS in LogFlusherConfig::from_env() - Update ship_batch to accept explicit api_key param so each additional endpoint uses its own key instead of the primary key - Re-export LogsAdditionalEndpoint from crate root - Update all test fixtures to use the new struct Rationale: aligns with the datadog-lambda-extension bottlecap model where each additional endpoint authenticates independently with its own API key * fix(log-agent): fall back to uncompressed on zstd compression failure - Replace ? propagation with match in ship_batch compression block - On compress error, warn and send raw bytes without Content-Encoding header - Avoids dropping the batch entirely due to a transient encoder failure Rationale: compression failures are rare (OOM, corrupted encoder state) and silently dropping the batch is worse than sending it uncompressed * feat(log-agent): implement cross-invocation retry via RequestBuilder passback - Change flush() -> bool to flush(Vec) -> Vec - send_with_retry returns Err(builder) on transient exhaustion instead of FlushError - serverless-compat flush loop stores and redrives failed builders each cycle - Additional endpoint failures remain best-effort (not tracked for retry) - Add tests: cross-invocation redrive succeeds, additional endpoint failures excluded Rationale: aligns with bottlecap FlushingService retry pattern; batches that hit transient intake errors survive across Lambda invocations instead of being silently dropped after MAX_FLUSH_ATTEMPTS This commit made by [/dd:git:commit:quick](https://github.com/DataDog/claude-marketplace/tree/main/dd/commands/git/commit/quick.md) * fix(log-agent): account for JSON framing bytes in batch size checks - Include `[`/`]` (2 bytes) and comma separators in `is_full()` and `get_batch()` overflow guards - Batch wire size could silently exceed MAX_CONTENT_BYTES by up to N+1 bytes Rationale: JSON array framing is part of the wire payload but was not counted in the 5 MB cap checks This commit made by [/dd:git:commit:quick](https://github.com/DataDog/claude-marketplace/tree/main/dd/commands/git/commit/quick.md) * test(log-agent): replace tautological OPW URL test with real behavior check - Old test only asserted String::new() is empty — never called production code - New test calls start_log_agent() with OPW enabled + empty URL and asserts None Rationale: the test was a no-op; it now exercises the actual guard in start_log_agent() This commit made by [/dd:git:commit:quick](https://github.com/DataDog/claude-marketplace/tree/main/dd/commands/git/commit/quick.md) * test(log-agent): remove no-op compile-time size_of test - Delete _assert_log_flusher_constructible and its unused imports - size_of checks never fail and provide no constructibility guarantee Rationale: cargo check and existing integration tests already cover API visibility; the dead function only created maintenance noise This commit made by [/dd:git:commit:quick](https://github.com/DataDog/claude-marketplace/tree/main/dd/commands/git/commit/quick.md) * refactor(log-agent): rename LogEntry::new to from_message; drop compression_level range from docs - Rename LogEntry::new → LogEntry::from_message and update all call sites - Remove inaccurate "1–21" range from compression_level doc comment Rationale: new() implies all fields are provided; from_message makes the partial construction explicit per Rust API Guidelines This commit made by [/dd:git:commit:quick](https://github.com/DataDog/claude-marketplace/tree/main/dd/commands/git/commit/quick.md) * refactor(log-agent): rename LogEntry → IntakeEntry and log_entry → intake_entry - Rename struct, file, and all call sites across the codebase - Name now references the Datadog Logs Intake API format explicitly Rationale: reviewer feedback — the name should reflect the Intake Log format; LogEntry was too generic This commit made by [/dd:git:commit:quick](https://github.com/DataDog/claude-marketplace/tree/main/dd/commands/git/commit/quick.md) * refactor(log-agent): drop Log prefix from Aggregator and AggregatorCommand - Rename LogAggregator → Aggregator and LogAggregatorCommand → AggregatorCommand - No need for Log prefix inside the logs crate Rationale: reviewer feedback — redundant Log prefix within datadog-log-agent crate This commit made by [/dd:git:commit:quick](https://github.com/DataDog/claude-marketplace/tree/main/dd/commands/git/commit/quick.md) * refactor(log-agent): rename FlusherMode → Destination; add #[must_use] to from_env - FlusherMode renamed to Destination — it describes where logs are sent, not how - #[must_use] added to LogFlusherConfig::from_env() to catch ignored return values Rationale: reviewer feedback — FlusherMode name is misleading; Destination is more accurate This commit made by [/dd:git:commit:quick](https://github.com/DataDog/claude-marketplace/tree/main/dd/commands/git/commit/quick.md) * chore: fix all clippy and fmt warnings across workspace - Wrap env::set_var/remove_var in unsafe blocks (Rust 2024 requirement) - Collapse nested if-let + if into let-chain patterns - Replace match Ok/Err with .err(), if let Err(_) with .is_err() - Use .is_multiple_of() and assert!(!…) idioms - Remove redundant as u32 casts on already-u32 fields - Suppress result_large_err for external figment::Error in test modules - Suppress disallowed_methods for reqwest::Client::builder in tests Rationale: cargo clippy --workspace --all-targets and cargo fmt reported errors and warnings that needed to be resolved This commit made by [/dd:git:commit:quick](https://github.com/DataDog/claude-marketplace/tree/main/dd/commands/git/commit/quick.md) * feat(datadog-log-agent): add LogEntry and FlusherMode type aliases - Add `pub type LogEntry = IntakeEntry` alias - Add `pub type FlusherMode = Destination` alias Rationale: bottlecap's datadog-log-agent adoption (SVLS-8573) was written against these names. The crate used IntakeEntry/Destination but never exported the aliases the plan specified, leaving bottlecap unable to compile against the crate. * revert(datadog-log-agent): remove LogEntry/FlusherMode type aliases Bottlecap now uses IntakeEntry and Destination directly, so the aliases are not needed. * refactor(log-agent): rename crate datadog-log-agent → datadog-logs-agent - Renamed crates/datadog-log-agent/ directory to crates/datadog-logs-agent/ - Updated package name in crate Cargo.toml - Updated dependency reference in datadog-serverless-compat/Cargo.toml - Updated all use datadog_log_agent:: identifiers in main.rs, examples, and tests - Updated references in scripts/test-log-intake.sh and AGENTS.md Rationale: Align crate name with the plural "logs" convention used elsewhere in the codebase. This commit made by [/dd:git:commit:quick](https://github.com/DataDog/claude-marketplace/tree/main/dd/commands/git/commit/quick.md) * fix(log-agent): fix OPW compression bleed and add TODO comments for follow-up fixes - flusher.rs: rename use_compression → primary_use_compression; pass config.use_compression to additional endpoints so they honour DD_LOGS_CONFIG_USE_COMPRESSION regardless of OPW primary destination. Add DD-PROTOCOL header via is_datadog_intake flag instead of checking mode directly, so additional endpoints always get the header. Add tests: test_opw_primary_additional_endpoint_receives_dd_protocol_header, test_opw_primary_additional_endpoint_compresses_when_enabled, test_opw_primary_never_compressed_even_when_flag_set. - server.rs: add TODO(SVLS-chunked-body-limit) describing the chunked transfer body-size bypass and the Limited-based fix to implement later. - main.rs: replace vague TODO with TODO(SVLS-bind-fail-fast) summarising the bind-fail-fast fix (BoundLogServer split + async start_log_agent). Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 --- .gitignore | 2 + Cargo.lock | 22 + crates/datadog-agent-config/env.rs | 1 + crates/datadog-agent-config/mod.rs | 1 + crates/datadog-agent-config/yaml.rs | 1 + crates/datadog-logs-agent/Cargo.toml | 35 + .../datadog-logs-agent/examples/send_logs.rs | 196 ++++ .../datadog-logs-agent/src/aggregator/core.rs | 270 ++++++ .../datadog-logs-agent/src/aggregator/mod.rs | 8 + .../src/aggregator/service.rs | 185 ++++ crates/datadog-logs-agent/src/config.rs | 109 +++ crates/datadog-logs-agent/src/constants.rs | 20 + crates/datadog-logs-agent/src/errors.rs | 35 + crates/datadog-logs-agent/src/flusher.rs | 891 ++++++++++++++++++ crates/datadog-logs-agent/src/intake_entry.rs | 188 ++++ crates/datadog-logs-agent/src/lib.rs | 26 + .../src/logs_additional_endpoint.rs | 104 ++ crates/datadog-logs-agent/src/server.rs | 523 ++++++++++ .../tests/integration_test.rs | 846 +++++++++++++++++ crates/datadog-serverless-compat/Cargo.toml | 6 + crates/datadog-serverless-compat/src/main.rs | 240 ++++- crates/datadog-trace-agent/src/aggregator.rs | 2 +- .../tests/integration_test.rs | 10 +- crates/dogstatsd/src/flusher.rs | 1 + crates/dogstatsd/src/origin.rs | 93 +- crates/dogstatsd/tests/integration_test.rs | 1 + scripts/test-log-intake.sh | 134 +++ 27 files changed, 3875 insertions(+), 75 deletions(-) create mode 100644 crates/datadog-logs-agent/Cargo.toml create mode 100644 crates/datadog-logs-agent/examples/send_logs.rs create mode 100644 crates/datadog-logs-agent/src/aggregator/core.rs create mode 100644 crates/datadog-logs-agent/src/aggregator/mod.rs create mode 100644 crates/datadog-logs-agent/src/aggregator/service.rs create mode 100644 crates/datadog-logs-agent/src/config.rs create mode 100644 crates/datadog-logs-agent/src/constants.rs create mode 100644 crates/datadog-logs-agent/src/errors.rs create mode 100644 crates/datadog-logs-agent/src/flusher.rs create mode 100644 crates/datadog-logs-agent/src/intake_entry.rs create mode 100644 crates/datadog-logs-agent/src/lib.rs create mode 100644 crates/datadog-logs-agent/src/logs_additional_endpoint.rs create mode 100644 crates/datadog-logs-agent/src/server.rs create mode 100644 crates/datadog-logs-agent/tests/integration_test.rs create mode 100755 scripts/test-log-intake.sh diff --git a/.gitignore b/.gitignore index 0dd34a5..15ed4dd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ /target /.idea /.worktrees +/CLAUDE.md +/AGENTS.md diff --git a/Cargo.lock b/Cargo.lock index 9a7c7be..8970e45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -467,6 +467,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "datadog-logs-agent" +version = "0.1.0" +dependencies = [ + "datadog-fips", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "mockito", + "reqwest", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tracing", + "zstd", +] + [[package]] name = "datadog-opentelemetry" version = "0.3.0" @@ -519,10 +539,12 @@ name = "datadog-serverless-compat" version = "0.1.0" dependencies = [ "datadog-fips", + "datadog-logs-agent", "datadog-trace-agent", "dogstatsd", "libdd-trace-utils 3.0.0", "reqwest", + "serde_json", "tokio", "tokio-util", "tracing", diff --git a/crates/datadog-agent-config/env.rs b/crates/datadog-agent-config/env.rs index f24d6be..4dae58b 100644 --- a/crates/datadog-agent-config/env.rs +++ b/crates/datadog-agent-config/env.rs @@ -707,6 +707,7 @@ impl ConfigSource for EnvConfigSource { #[cfg_attr(coverage_nightly, coverage(off))] // Test modules skew coverage metrics #[cfg(test)] +#[allow(clippy::result_large_err)] mod tests { use std::time::Duration; diff --git a/crates/datadog-agent-config/mod.rs b/crates/datadog-agent-config/mod.rs index 6fc858a..e8a29bb 100644 --- a/crates/datadog-agent-config/mod.rs +++ b/crates/datadog-agent-config/mod.rs @@ -881,6 +881,7 @@ where #[cfg_attr(coverage_nightly, coverage(off))] // Test modules skew coverage metrics #[cfg(test)] +#[allow(clippy::result_large_err)] pub mod tests { use libdd_trace_obfuscation::replacer::parse_rules_from_string; diff --git a/crates/datadog-agent-config/yaml.rs b/crates/datadog-agent-config/yaml.rs index 06b7851..2c9d49f 100644 --- a/crates/datadog-agent-config/yaml.rs +++ b/crates/datadog-agent-config/yaml.rs @@ -770,6 +770,7 @@ impl ConfigSource for YamlConfigSource { #[cfg_attr(coverage_nightly, coverage(off))] // Test modules skew coverage metrics #[cfg(test)] +#[allow(clippy::result_large_err)] mod tests { use std::path::Path; use std::time::Duration; diff --git a/crates/datadog-logs-agent/Cargo.toml b/crates/datadog-logs-agent/Cargo.toml new file mode 100644 index 0000000..177d561 --- /dev/null +++ b/crates/datadog-logs-agent/Cargo.toml @@ -0,0 +1,35 @@ +# Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "datadog-logs-agent" +version = "0.1.0" +edition.workspace = true +license.workspace = true + +[lib] +bench = false + +[dependencies] +datadog-fips = { path = "../datadog-fips" } +reqwest = { version = "0.12.4", features = ["json", "http2"], default-features = false } +serde = { version = "1.0.197", default-features = false, features = ["derive"] } +serde_json = { version = "1.0.116", default-features = false, features = ["alloc"] } +thiserror = { version = "1.0.58", default-features = false } +hyper = { version = "1", features = ["http1", "server"] } +http-body-util = "0.1" +hyper-util = { version = "0.1", features = ["tokio"] } +futures = { version = "0.3", default-features = false, features = ["alloc"] } +tokio = { version = "1.37.0", default-features = false, features = ["sync", "net"] } +tracing = { version = "0.1.40", default-features = false } +zstd = { version = "0.13.3", default-features = false } + +[dev-dependencies] +http = "1" +mockito = { version = "1.5.0", default-features = false } +serde_json = { version = "1.0.116", default-features = false, features = ["alloc"] } +reqwest = { version = "0.12.4", features = ["json"], default-features = false } +tokio = { version = "1.37.0", default-features = false, features = ["macros", "rt-multi-thread", "net", "time"] } + +[features] +default = [] diff --git a/crates/datadog-logs-agent/examples/send_logs.rs b/crates/datadog-logs-agent/examples/send_logs.rs new file mode 100644 index 0000000..7d64df9 --- /dev/null +++ b/crates/datadog-logs-agent/examples/send_logs.rs @@ -0,0 +1,196 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Local test helper: inserts sample log entries and flushes them via the log agent pipeline. +//! +//! # Usage +//! +//! ## Flush to a local capture server (recommended for local dev) +//! +//! The easiest way — runs capture server and example together: +//! ./scripts/test-log-intake.sh +//! +//! Or manually in two terminals: +//! +//! In terminal 1 — start the capture server (handles POST, prints JSON): +//! python3 scripts/test-log-intake.sh # not available standalone +//! # Use the script above, or run: python3 -c "$(sed -n '/PYEOF/,/PYEOF/p' scripts/test-log-intake.sh)" +//! +//! In terminal 2 — run this example: +//! DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED=true \ +//! DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL=http://localhost:9999/logs \ +//! DD_API_KEY=local-test-key \ +//! cargo run -p datadog-logs-agent --example send_logs +//! +//! NOTE: `python3 -m http.server` does NOT work — it rejects POST requests. +//! +//! ## Flush to a real Datadog endpoint +//! +//! DD_API_KEY= \ +//! DD_SITE=datadoghq.com \ +//! cargo run -p datadog-logs-agent --example send_logs +//! +//! ## Configuration via env vars +//! +//! | Variable | Default | +//! |--------------------------------------------------|--------------------| +//! | DD_API_KEY | (empty) | +//! | DD_SITE | datadoghq.com | +//! | DD_LOGS_CONFIG_USE_COMPRESSION | true | +//! | DD_LOGS_CONFIG_COMPRESSION_LEVEL | 3 | +//! | DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED | false | +//! | DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL | (empty) | +//! | LOG_ENTRY_COUNT | 5 | + +use datadog_logs_agent::{ + AggregatorService, Destination, IntakeEntry, LogFlusher, LogFlusherConfig, +}; + +#[allow(clippy::disallowed_methods)] // plain reqwest::Client for local testing +#[tokio::main] +async fn main() { + let entry_count: usize = std::env::var("LOG_ENTRY_COUNT") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(5); + + let config = LogFlusherConfig::from_env(); + + // Print effective configuration + let (endpoint, compressed) = describe_config(&config); + println!("──────────────────────────────────────────"); + println!(" datadog-logs-agent local test"); + println!("──────────────────────────────────────────"); + println!(" endpoint : {endpoint}"); + println!(" api_key : {}", mask(&config.api_key)); + println!(" compressed : {compressed}"); + println!(" entries : {entry_count}"); + println!("──────────────────────────────────────────"); + + // Start aggregator service + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + // Insert sample entries representing different runtimes + let mut entries = Vec::with_capacity(entry_count); + + for i in 0..entry_count { + let entry = match i % 3 { + 0 => lambda_entry(i), + 1 => azure_entry(i), + _ => plain_entry(i), + }; + entries.push(entry); + } + + println!("\nInserting {entry_count} log entries..."); + handle.insert_batch(entries).expect("insert_batch failed"); + + // Build HTTP client + let client = reqwest::Client::builder() + .timeout(config.flush_timeout) + .build() + .expect("failed to build HTTP client"); + + // Flush + println!("Flushing to {endpoint}..."); + let flusher = LogFlusher::new(config, client, handle); + let failed = flusher.flush(vec![]).await; + + if failed.is_empty() { + println!("\n✓ Flush succeeded"); + } else { + eprintln!("\n✗ Flush failed — check endpoint and API key"); + std::process::exit(1); + } +} + +// ── Sample log entry builders ───────────────────────────────────────────────── + +fn lambda_entry(i: usize) -> IntakeEntry { + let mut attrs = serde_json::Map::new(); + attrs.insert( + "lambda".to_string(), + serde_json::json!({ + "arn": "arn:aws:lambda:us-east-1:123456789012:function:my-fn", + "request_id": format!("req-{i:04}") + }), + ); + IntakeEntry { + message: format!("[lambda] invocation #{i} completed"), + timestamp: now_ms(), + hostname: Some("arn:aws:lambda:us-east-1:123456789012:function:my-fn".to_string()), + service: Some("my-fn".to_string()), + ddsource: Some("lambda".to_string()), + ddtags: Some("env:local,runtime:lambda".to_string()), + status: Some("info".to_string()), + attributes: attrs, + } +} + +fn azure_entry(i: usize) -> IntakeEntry { + let mut attrs = serde_json::Map::new(); + attrs.insert( + "azure".to_string(), + serde_json::json!({ + "resource_id": "/subscriptions/sub-123/resourceGroups/rg/providers/Microsoft.Web/sites/my-fn", + "operation_name": "Microsoft.Web/sites/functions/run/action" + }), + ); + IntakeEntry { + message: format!("[azure] function triggered #{i}"), + timestamp: now_ms(), + hostname: Some("my-azure-fn".to_string()), + service: Some("payments".to_string()), + ddsource: Some("azure-functions".to_string()), + ddtags: Some("env:local,runtime:azure".to_string()), + status: Some("info".to_string()), + attributes: attrs, + } +} + +fn plain_entry(i: usize) -> IntakeEntry { + IntakeEntry { + message: format!("[generic] log message #{i}"), + timestamp: now_ms(), + hostname: Some("localhost".to_string()), + service: Some("test-service".to_string()), + ddsource: Some("rust".to_string()), + ddtags: Some("env:local".to_string()), + status: if i.is_multiple_of(5) { + Some("error".to_string()) + } else { + Some("info".to_string()) + }, + attributes: serde_json::Map::new(), + } +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +fn now_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as i64) + .unwrap_or(0) +} + +fn describe_config(config: &LogFlusherConfig) -> (String, bool) { + match &config.mode { + Destination::Datadog => ( + format!("https://http-intake.logs.{}/api/v2/logs", config.site), + config.use_compression, + ), + Destination::ObservabilityPipelinesWorker { url } => (url.clone(), false), + } +} + +fn mask(s: &str) -> String { + if s.is_empty() { + return "(not set)".to_string(); + } + if s.len() <= 8 { + return "*".repeat(s.len()); + } + format!("{}…{}", &s[..4], &s[s.len() - 4..]) +} diff --git a/crates/datadog-logs-agent/src/aggregator/core.rs b/crates/datadog-logs-agent/src/aggregator/core.rs new file mode 100644 index 0000000..97a64f4 --- /dev/null +++ b/crates/datadog-logs-agent/src/aggregator/core.rs @@ -0,0 +1,270 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::VecDeque; + +use crate::constants::{MAX_BATCH_ENTRIES, MAX_CONTENT_BYTES, MAX_LOG_BYTES}; +use crate::errors::AggregatorError; +use crate::intake_entry::IntakeEntry; + +/// In-memory log batch accumulator. +/// +/// Stores pre-serialized JSON strings in a FIFO queue. A batch is "full" +/// when it reaches `MAX_BATCH_ENTRIES` entries or `MAX_CONTENT_BYTES` of +/// uncompressed content. +pub struct Aggregator { + messages: VecDeque, + current_size_bytes: usize, +} + +impl Default for Aggregator { + fn default() -> Self { + Self::new() + } +} + +impl Aggregator { + /// Create a new, empty aggregator. + pub fn new() -> Self { + Self { + messages: VecDeque::new(), + current_size_bytes: 0, + } + } + + /// Insert a log entry into the batch. + /// + /// Returns `Ok(true)` if the batch is now full and ready to flush. + /// Returns `Err(AggregatorError::EntryTooLarge)` if the serialized + /// entry exceeds `MAX_LOG_BYTES` — the entry is dropped. + pub fn insert(&mut self, entry: &IntakeEntry) -> Result { + let serialized = serde_json::to_string(entry)?; + let len = serialized.len(); + + if len > MAX_LOG_BYTES { + return Err(AggregatorError::EntryTooLarge { + size: len, + max: MAX_LOG_BYTES, + }); + } + + self.messages.push_back(serialized); + self.current_size_bytes += len; + + Ok(self.is_full()) + } + + /// Returns `true` if the batch has reached its entry count or byte limit. + /// + /// The byte check accounts for JSON framing: `[` + `]` (2 bytes) plus one + /// comma per entry after the first (`N - 1` bytes). + pub fn is_full(&self) -> bool { + let n = self.messages.len(); + // framing: 2 bytes for `[`/`]` + (n - 1) commas + let framing = if n == 0 { 0 } else { 2 + (n - 1) }; + n >= MAX_BATCH_ENTRIES || self.current_size_bytes + framing >= MAX_CONTENT_BYTES + } + + /// Returns `true` if no log entries are buffered. + pub fn is_empty(&self) -> bool { + self.messages.is_empty() + } + + /// Returns the number of log entries currently buffered. + pub fn len(&self) -> usize { + self.messages.len() + } + + /// Drain up to `MAX_BATCH_ENTRIES` entries and return them as a JSON + /// array (`[entry1,entry2,...]`) encoded as UTF-8 bytes. + /// + /// Returns `None` if the aggregator is empty. + pub fn get_batch(&mut self) -> Option> { + if self.messages.is_empty() { + return None; + } + + let mut output = Vec::new(); + output.push(b'['); + let mut bytes_in_batch: usize = 0; + let mut count: usize = 0; + + loop { + if count >= MAX_BATCH_ENTRIES { + break; + } + + let msg_len = match self.messages.front() { + Some(m) => m.len(), + None => break, + }; + + // Account for the comma separator and the 2-byte `[`/`]` framing. + // Total wire size = bytes_in_batch + (separator) + msg_len + 2 (for `[` and `]`) + let separator = if count == 0 { 0 } else { 1 }; + if bytes_in_batch + separator + msg_len + 2 > MAX_CONTENT_BYTES { + break; + } + + // Safe: we just confirmed front() is Some and we hold &mut self + let msg = match self.messages.pop_front() { + Some(m) => m, + None => break, + }; + + if count > 0 { + output.push(b','); + bytes_in_batch += 1; + } + + self.current_size_bytes = self.current_size_bytes.saturating_sub(msg.len()); + bytes_in_batch += msg.len(); + count += 1; + output.extend_from_slice(msg.as_bytes()); + } + + output.push(b']'); + Some(output) + } + + /// Drain all entries and return them as a `Vec` of JSON array batches. + /// May return multiple batches if the queue exceeds a single batch limit. + pub fn get_all_batches(&mut self) -> Vec> { + let mut batches = Vec::new(); + while let Some(batch) = self.get_batch() { + batches.push(batch); + } + batches + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::intake_entry::IntakeEntry; + + fn make_entry(msg: &str) -> IntakeEntry { + IntakeEntry::from_message(msg, 1_700_000_000_000) + } + + #[test] + fn test_new_aggregator_is_empty() { + let agg = Aggregator::new(); + assert!(agg.is_empty()); + assert_eq!(agg.len(), 0); + } + + #[test] + fn test_insert_single_entry() { + let mut agg = Aggregator::new(); + let full = agg.insert(&make_entry("hello")).expect("insert failed"); + assert!(!full, "single entry should not fill the batch"); + assert_eq!(agg.len(), 1); + assert!(!agg.is_empty()); + } + + #[test] + fn test_get_batch_returns_valid_json_array() { + let mut agg = Aggregator::new(); + agg.insert(&make_entry("line 1")).expect("insert"); + agg.insert(&make_entry("line 2")).expect("insert"); + + let batch = agg.get_batch().expect("should have a batch"); + let parsed: serde_json::Value = serde_json::from_slice(&batch).expect("valid JSON"); + + assert!(parsed.is_array()); + assert_eq!(parsed.as_array().unwrap().len(), 2); + assert_eq!(parsed[0]["message"], "line 1"); + assert_eq!(parsed[1]["message"], "line 2"); + } + + #[test] + fn test_get_batch_drains_aggregator() { + let mut agg = Aggregator::new(); + agg.insert(&make_entry("log")).expect("insert"); + + let _ = agg.get_batch(); + assert!(agg.is_empty(), "aggregator should be empty after get_batch"); + assert!( + agg.get_batch().is_none(), + "second get_batch should return None" + ); + } + + #[test] + fn test_entry_too_large_returns_error() { + let mut agg = Aggregator::new(); + // Construct an entry whose JSON serialization exceeds MAX_LOG_BYTES + let big_message = "x".repeat(crate::constants::MAX_LOG_BYTES + 1); + let entry = make_entry(&big_message); + let result = agg.insert(&entry); + assert!(result.is_err(), "oversized entry should be rejected"); + assert!( + agg.is_empty(), + "aggregator should stay empty after rejection" + ); + } + + #[test] + fn test_insert_returns_true_when_batch_full_by_count() { + use crate::constants::MAX_BATCH_ENTRIES; + let mut agg = Aggregator::new(); + let entry = make_entry("x"); + + for _ in 0..(MAX_BATCH_ENTRIES - 1) { + let full = agg.insert(&entry).expect("insert"); + assert!(!full, "should not be full until last entry"); + } + let full = agg.insert(&entry).expect("insert"); + assert!(full, "should be full after MAX_BATCH_ENTRIES entries"); + assert!(agg.is_full()); + } + + #[test] + fn test_get_all_batches_splits_large_queue() { + use crate::constants::MAX_BATCH_ENTRIES; + let mut agg = Aggregator::new(); + let entry = make_entry("x"); + + for _ in 0..(MAX_BATCH_ENTRIES + 5) { + let _ = agg.insert(&entry); + } + + let batches = agg.get_all_batches(); + assert_eq!(batches.len(), 2, "should produce 2 batches"); + + let first: serde_json::Value = serde_json::from_slice(&batches[0]).expect("json"); + let second: serde_json::Value = serde_json::from_slice(&batches[1]).expect("json"); + assert_eq!(first.as_array().unwrap().len(), MAX_BATCH_ENTRIES); + assert_eq!(second.as_array().unwrap().len(), 5); + } + + #[test] + fn test_get_all_batches_empty_returns_empty_vec() { + let mut agg = Aggregator::new(); + assert!(agg.get_all_batches().is_empty()); + } + + #[test] + fn test_batch_never_exceeds_max_content_bytes() { + // Fill with entries whose sizes sum to just under MAX_CONTENT_BYTES so + // that the framing bytes (`[`, `]`, commas) would push a naive + // implementation over the limit. + let mut agg = Aggregator::new(); + // Each entry's serialized JSON is roughly 50 bytes; pack enough entries + // that their raw sum approaches MAX_CONTENT_BYTES. + let entry = make_entry("x"); + for _ in 0..1000 { + let _ = agg.insert(&entry); + } + + for batch in agg.get_all_batches() { + assert!( + batch.len() <= MAX_CONTENT_BYTES, + "batch size {} exceeds MAX_CONTENT_BYTES {}", + batch.len(), + MAX_CONTENT_BYTES + ); + } + } +} diff --git a/crates/datadog-logs-agent/src/aggregator/mod.rs b/crates/datadog-logs-agent/src/aggregator/mod.rs new file mode 100644 index 0000000..c191ab6 --- /dev/null +++ b/crates/datadog-logs-agent/src/aggregator/mod.rs @@ -0,0 +1,8 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +pub mod core; +pub use core::Aggregator; + +pub mod service; +pub use service::{AggregatorHandle, AggregatorService}; diff --git a/crates/datadog-logs-agent/src/aggregator/service.rs b/crates/datadog-logs-agent/src/aggregator/service.rs new file mode 100644 index 0000000..c4ffc64 --- /dev/null +++ b/crates/datadog-logs-agent/src/aggregator/service.rs @@ -0,0 +1,185 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use tokio::sync::{mpsc, oneshot}; +use tracing::{debug, error, warn}; + +use crate::aggregator::Aggregator; +use crate::intake_entry::IntakeEntry; + +#[derive(Debug)] +enum AggregatorCommand { + InsertBatch(Vec), + GetBatches(oneshot::Sender>>), + Shutdown, +} + +/// Cloneable handle for sending commands to a running [`AggregatorService`]. +#[derive(Clone)] +pub struct AggregatorHandle { + tx: mpsc::UnboundedSender, +} + +impl AggregatorHandle { + /// Queue a batch of log entries for aggregation. + /// + /// Returns an error only if the service has already stopped. + pub fn insert_batch(&self, entries: Vec) -> Result<(), String> { + self.tx + .send(AggregatorCommand::InsertBatch(entries)) + .map_err(|e| format!("failed to send InsertBatch: {e}")) + } + + /// Retrieve and drain all accumulated log batches as JSON arrays. + /// + /// Returns an empty `Vec` if the aggregator holds no logs. + pub async fn get_batches(&self) -> Result>, String> { + let (tx, rx) = oneshot::channel(); + self.tx + .send(AggregatorCommand::GetBatches(tx)) + .map_err(|e| format!("failed to send GetBatches: {e}"))?; + rx.await + .map_err(|e| format!("failed to receive GetBatches response: {e}")) + } + + /// Signal the service to stop processing and exit its run loop. + pub fn shutdown(&self) -> Result<(), String> { + self.tx + .send(AggregatorCommand::Shutdown) + .map_err(|e| format!("failed to send Shutdown: {e}")) + } +} + +/// Background tokio task owning a [`Aggregator`] and processing commands. +/// +/// Create with [`AggregatorService::new`], spawn with `tokio::spawn(service.run())`, +/// and interact via the returned [`AggregatorHandle`]. +pub struct AggregatorService { + aggregator: Aggregator, + rx: mpsc::UnboundedReceiver, +} + +impl AggregatorService { + /// Create a new service and its associated handle. + pub fn new() -> (Self, AggregatorHandle) { + let (tx, rx) = mpsc::unbounded_channel(); + let service = Self { + aggregator: Aggregator::new(), + rx, + }; + let handle = AggregatorHandle { tx }; + (service, handle) + } + + /// Run the service event loop. + /// + /// Returns when a `Shutdown` command is received or the last handle is dropped. + pub async fn run(mut self) { + debug!("log aggregator service started"); + + while let Some(command) = self.rx.recv().await { + match command { + AggregatorCommand::InsertBatch(entries) => { + for entry in &entries { + if let Err(e) = self.aggregator.insert(entry) { + warn!("dropping log entry: {e}"); + } + } + } + + AggregatorCommand::GetBatches(response_tx) => { + let batches = self.aggregator.get_all_batches(); + if response_tx.send(batches).is_err() { + error!("failed to send GetBatches response — receiver dropped"); + } + } + + AggregatorCommand::Shutdown => { + debug!("log aggregator service shutting down"); + break; + } + } + } + + debug!("log aggregator service stopped"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::intake_entry::IntakeEntry; + + fn make_entry(msg: &str) -> IntakeEntry { + IntakeEntry::from_message(msg, 1_700_000_000_000) + } + + #[tokio::test] + async fn test_insert_and_get_batches_roundtrip() { + let (service, handle) = AggregatorService::new(); + let task = tokio::spawn(service.run()); + + handle + .insert_batch(vec![make_entry("a"), make_entry("b")]) + .expect("insert_batch failed"); + + let batches = handle.get_batches().await.expect("get_batches failed"); + assert_eq!(batches.len(), 1); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("json"); + assert_eq!(arr.as_array().unwrap().len(), 2); + + handle.shutdown().expect("shutdown failed"); + task.await.expect("task panicked"); + } + + #[tokio::test] + async fn test_get_batches_empty_returns_empty_vec() { + let (service, handle) = AggregatorService::new(); + let task = tokio::spawn(service.run()); + + let batches = handle.get_batches().await.expect("get_batches"); + assert!(batches.is_empty()); + + handle.shutdown().expect("shutdown"); + task.await.expect("task"); + } + + #[tokio::test] + async fn test_oversized_entry_dropped_not_panicked() { + let (service, handle) = AggregatorService::new(); + let task = tokio::spawn(service.run()); + + let big = IntakeEntry::from_message("x".repeat(crate::constants::MAX_LOG_BYTES + 1), 0); + handle.insert_batch(vec![big]).expect("send ok"); + + let batches = handle.get_batches().await.expect("get_batches"); + assert!( + batches.is_empty(), + "oversized entry should have been dropped" + ); + + handle.shutdown().expect("shutdown"); + task.await.expect("task"); + } + + #[tokio::test] + async fn test_handle_is_clone_and_both_can_insert() { + let (service, handle) = AggregatorService::new(); + let task = tokio::spawn(service.run()); + + let handle2 = handle.clone(); + handle + .insert_batch(vec![make_entry("from h1")]) + .expect("h1"); + handle2 + .insert_batch(vec![make_entry("from h2")]) + .expect("h2"); + + let batches = handle.get_batches().await.expect("get_batches"); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("json"); + assert_eq!(arr.as_array().unwrap().len(), 2); + + handle.shutdown().expect("shutdown"); + task.await.expect("task"); + } +} diff --git a/crates/datadog-logs-agent/src/config.rs b/crates/datadog-logs-agent/src/config.rs new file mode 100644 index 0000000..630e686 --- /dev/null +++ b/crates/datadog-logs-agent/src/config.rs @@ -0,0 +1,109 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::time::Duration; + +use crate::constants::{DEFAULT_COMPRESSION_LEVEL, DEFAULT_FLUSH_TIMEOUT_SECS, DEFAULT_SITE}; +use crate::logs_additional_endpoint::{LogsAdditionalEndpoint, parse_additional_endpoints}; + +/// Controls where and how logs are shipped. +#[derive(Debug, Clone)] +pub enum Destination { + /// Ship to Datadog Logs API. + /// Endpoint: `https://http-intake.logs.{site}/api/v2/logs` + /// Headers: `DD-API-KEY`, `DD-PROTOCOL: agent-json`, optionally `Content-Encoding: zstd` + Datadog, + + /// Ship to an Observability Pipelines Worker. + /// Endpoint: the provided URL. + /// Headers: `DD-API-KEY` only. Compression is always disabled for OPW. + ObservabilityPipelinesWorker { url: String }, +} + +/// Configuration for [`LogFlusher`](crate::flusher::LogFlusher). +#[derive(Debug, Clone)] +pub struct LogFlusherConfig { + /// Datadog API key. + pub api_key: String, + + /// Datadog site (e.g. "datadoghq.com", "datadoghq.eu"). + pub site: String, + + /// Flusher mode — Datadog vs Observability Pipelines Worker. + pub mode: Destination, + + /// Additional Datadog intake endpoints to ship each batch to in parallel. + /// Each endpoint uses its own API key and full intake URL. + pub additional_endpoints: Vec, + + /// Enable zstd compression (ignored in OPW mode, which is always uncompressed). + pub use_compression: bool, + + /// zstd compression level (ignored when `use_compression` is false). + pub compression_level: i32, + + /// Per-request timeout. + pub flush_timeout: Duration, +} + +impl LogFlusherConfig { + /// Build a config from environment variables, falling back to sensible defaults. + /// + /// | Variable | Default | + /// |---|---| + /// | `DD_API_KEY` | `""` | + /// | `DD_SITE` | `datadoghq.com` | + /// | `DD_LOGS_CONFIG_USE_COMPRESSION` | `true` | + /// | `DD_LOGS_CONFIG_COMPRESSION_LEVEL` | `3` | + /// | `DD_FLUSH_TIMEOUT` | `5` (seconds) | + /// | `DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED` | `false` | + /// | `DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL` | (none) | + #[must_use] + pub fn from_env() -> Self { + let api_key = std::env::var("DD_API_KEY").unwrap_or_default(); + let site = std::env::var("DD_SITE").unwrap_or_else(|_| DEFAULT_SITE.to_string()); + + let use_compression = std::env::var("DD_LOGS_CONFIG_USE_COMPRESSION") + .map(|v| v.to_lowercase() != "false") + .unwrap_or(true); + + let compression_level = std::env::var("DD_LOGS_CONFIG_COMPRESSION_LEVEL") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_COMPRESSION_LEVEL); + + let flush_timeout_secs = std::env::var("DD_FLUSH_TIMEOUT") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_FLUSH_TIMEOUT_SECS); + + let opw_enabled = std::env::var("DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED") + .map(|v| v.to_lowercase() == "true") + .unwrap_or(false); + + let mode = if opw_enabled { + let url = + std::env::var("DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL").unwrap_or_default(); + if url.is_empty() { + tracing::warn!( + "OPW mode enabled but DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL is not set — log flush will fail" + ); + } + Destination::ObservabilityPipelinesWorker { url } + } else { + Destination::Datadog + }; + + Self { + api_key, + site, + mode, + additional_endpoints: std::env::var("DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS") + .map(|v| parse_additional_endpoints(&v)) + .unwrap_or_default(), + use_compression, + compression_level, + flush_timeout: Duration::from_secs(flush_timeout_secs), + } + } +} diff --git a/crates/datadog-logs-agent/src/constants.rs b/crates/datadog-logs-agent/src/constants.rs new file mode 100644 index 0000000..018dd51 --- /dev/null +++ b/crates/datadog-logs-agent/src/constants.rs @@ -0,0 +1,20 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +/// Maximum number of log entries per batch. +pub const MAX_BATCH_ENTRIES: usize = 1_000; + +/// Maximum total uncompressed payload size per batch (5 MB). +pub const MAX_CONTENT_BYTES: usize = 5 * 1_024 * 1_024; + +/// Maximum allowed size for a single serialized log entry (1 MB). +pub const MAX_LOG_BYTES: usize = 1_024 * 1_024; + +/// Default Datadog site for log intake. +pub const DEFAULT_SITE: &str = "datadoghq.com"; + +/// Default flush timeout in seconds. +pub const DEFAULT_FLUSH_TIMEOUT_SECS: u64 = 5; + +/// Negative values enable ultra-fast modes. Level 3 is the zstd library default. +pub const DEFAULT_COMPRESSION_LEVEL: i32 = 3; diff --git a/crates/datadog-logs-agent/src/errors.rs b/crates/datadog-logs-agent/src/errors.rs new file mode 100644 index 0000000..50601af --- /dev/null +++ b/crates/datadog-logs-agent/src/errors.rs @@ -0,0 +1,35 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +/// Errors that can occur when inserting log entries into the aggregator. +#[derive(Debug, thiserror::Error)] +pub enum AggregatorError { + #[error("log entry too large: {size} bytes exceeds max {max} bytes")] + EntryTooLarge { size: usize, max: usize }, + + #[error("failed to serialize log entry: {0}")] + Serialization(#[from] serde_json::Error), +} + +/// Errors that can occur when flushing logs to Datadog. +#[derive(Debug, thiserror::Error)] +pub enum FlushError { + #[error("HTTP request failed: {0}")] + Request(String), + + #[error("server returned permanent error: status {status}")] + PermanentError { status: u16 }, + + #[error("max retries exceeded after {attempts} attempts")] + MaxRetriesExceeded { attempts: u32 }, + + #[error("compression failed: {0}")] + Compression(String), +} + +/// Errors that can occur during crate object creation. +#[derive(Debug, thiserror::Error)] +pub enum CreationError { + #[error("failed to build HTTP client: {0}")] + HttpClient(String), +} diff --git a/crates/datadog-logs-agent/src/flusher.rs b/crates/datadog-logs-agent/src/flusher.rs new file mode 100644 index 0000000..933ac5e --- /dev/null +++ b/crates/datadog-logs-agent/src/flusher.rs @@ -0,0 +1,891 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::io::Write as _; + +use futures::future::join_all; +use reqwest::Client; +use tracing::{debug, error, warn}; +use zstd::stream::write::Encoder; + +use crate::aggregator::AggregatorHandle; +use crate::config::{Destination, LogFlusherConfig}; +use crate::errors::FlushError; + +/// Maximum number of send attempts before giving up on a batch. +const MAX_FLUSH_ATTEMPTS: u32 = 3; + +/// Drains log batches from an [`AggregatorHandle`] and ships them to Datadog. +#[derive(Clone)] +pub struct LogFlusher { + config: LogFlusherConfig, + client: Client, + aggregator_handle: AggregatorHandle, +} + +impl LogFlusher { + /// Create a new flusher. + /// + /// The `client` **must** be built via + /// [`datadog_fips::reqwest_adapter::create_reqwest_client_builder`] to ensure + /// FIPS-compliant TLS. Never use `reqwest::Client::builder()` directly. + pub fn new( + config: LogFlusherConfig, + client: Client, + aggregator_handle: AggregatorHandle, + ) -> Self { + Self { + config, + client, + aggregator_handle, + } + } + + /// Drain the aggregator, ship all pending batches to Datadog, and redrive any + /// builders that failed transiently in the previous invocation. + /// + /// # Arguments + /// + /// * `retry_requests` — builders returned by a previous `flush` call that + /// exhausted their per-invocation retry budget. They are re-sent before + /// draining new batches from the aggregator. + /// + /// # Returns + /// + /// A vec of `RequestBuilder`s that still failed after all in-call retries. + /// The caller should pass these back on the next invocation to re-attempt + /// delivery. An empty vec means every batch was delivered successfully + /// (or encountered a permanent error and was dropped — those are logged). + /// + /// Failures on additional endpoints are logged as warnings but their + /// builders are not included in the returned vec (best-effort delivery). + pub async fn flush( + &self, + retry_requests: Vec, + ) -> Vec { + let mut failed: Vec = Vec::new(); + + // Redrive builders that failed transiently in the previous invocation. + if !retry_requests.is_empty() { + debug!( + "redriving {} log builder(s) from previous flush", + retry_requests.len() + ); + } + let retry_futures = retry_requests + .into_iter() + .map(|builder| async move { self.send_with_retry(builder).await.err() }); + for b in join_all(retry_futures).await.into_iter().flatten() { + failed.push(b); + } + + // Drain new batches from the aggregator. + let batches = match self.aggregator_handle.get_batches().await { + Ok(b) => b, + Err(e) => { + error!("failed to retrieve log batches from aggregator: {e}"); + return failed; + } + }; + + if batches.is_empty() { + debug!("no log batches to flush"); + return failed; + } + + debug!("flushing {} log batch(es)", batches.len()); + + let (primary_url, primary_use_compression) = self.resolve_endpoint(); + + let batch_futures = batches.iter().map(|batch| { + let primary_url = primary_url.clone(); + async move { + // Primary endpoint — failures are tracked for cross-invocation retry. + let is_primary_datadog = matches!(self.config.mode, Destination::Datadog); + let primary_result = self + .ship_batch(batch, &primary_url, primary_use_compression, &self.config.api_key, is_primary_datadog) + .await; + + // Additional endpoints — best-effort; failures are only logged. + // Additional endpoints are always Datadog intakes regardless of the + // primary destination, so DD-PROTOCOL: agent-json is always required. + // They use config.use_compression independently of the primary: OPW + // primaries disable compression for themselves but Datadog extras + // should still honour DD_LOGS_CONFIG_USE_COMPRESSION. + let extra_futures = self.config.additional_endpoints.iter().map(|endpoint| { + let url = endpoint.url.clone(); + let api_key = endpoint.api_key.clone(); + async move { + if self + .ship_batch(batch, &url, self.config.use_compression, &api_key, true) + .await + .is_err() + { + warn!( + "failed to ship log batch to additional endpoint {url} after all retries" + ); + } + } + }); + join_all(extra_futures).await; + + primary_result + } + }); + + for result in join_all(batch_futures).await { + if let Err(b) = result { + failed.push(b); + } + } + + failed + } + + fn resolve_endpoint(&self) -> (String, bool) { + match &self.config.mode { + Destination::Datadog => { + let url = format!("https://http-intake.logs.{}/api/v2/logs", self.config.site); + (url, self.config.use_compression) + } + Destination::ObservabilityPipelinesWorker { url } => { + // OPW does not support compression + (url.clone(), false) + } + } + } + + async fn ship_batch( + &self, + batch: &[u8], + url: &str, + compress: bool, + api_key: &str, + is_datadog_intake: bool, + ) -> Result<(), reqwest::RequestBuilder> { + let (body, content_encoding) = if compress { + match compress_zstd(batch, self.config.compression_level) { + Ok(compressed) => (compressed, Some("zstd")), + Err(e) => { + warn!("failed to compress log batch, sending uncompressed: {e}"); + (batch.to_vec(), None) + } + } + } else { + (batch.to_vec(), None) + }; + + let mut req = self + .client + .post(url) + .timeout(self.config.flush_timeout) + .header("DD-API-KEY", api_key) + .header("Content-Type", "application/json"); + + if is_datadog_intake { + req = req.header("DD-PROTOCOL", "agent-json"); + } + + if let Some(enc) = content_encoding { + req = req.header("Content-Encoding", enc); + } + + let req = req.body(body); + self.send_with_retry(req).await + } + + /// Send `builder`, retrying transient failures up to `MAX_FLUSH_ATTEMPTS`. + /// + /// # Returns + /// + /// * `Ok(())` — success **or** a permanent error (no point retrying; already + /// logged at `warn!`). + /// * `Err(builder)` — all attempts exhausted on a transient error. The + /// original builder is returned so the caller can retry it next invocation. + async fn send_with_retry( + &self, + builder: reqwest::RequestBuilder, + ) -> Result<(), reqwest::RequestBuilder> { + let mut attempts: u32 = 0; + + loop { + attempts += 1; + + let cloned = match builder.try_clone() { + Some(b) => b, + None => { + // Streaming body — can't clone, can't retry. + warn!("log batch request is not cloneable; dropping batch"); + return Ok(()); + } + }; + + match cloned.send().await { + Ok(resp) => { + let status = resp.status(); + // Drain the body so the underlying TCP connection is + // returned to the pool rather than held in CLOSE_WAIT. + let _ = resp.bytes().await; + + if status.is_success() { + debug!("log batch accepted: {status}"); + return Ok(()); + } + + // Retryable 4xx: treat like transient server errors and + // fall through to the retry loop below. + // 408 = Request Timeout (transient network condition) + // 425 = Too Early (TLS 0-RTT replay rejection) + // 429 = Too Many Requests (intake rate-limiting) + // + // TODO: for 429, parse the `Retry-After` response header + // and sleep for the indicated duration before retrying + // instead of retrying immediately, to avoid hammering the + // intake endpoint while it is still rate-limiting us. + let retryable_4xx = matches!(status.as_u16(), 408 | 425 | 429); + + // Permanent client errors — stop immediately, do not retry. + if status.as_u16() >= 400 && status.as_u16() < 500 && !retryable_4xx { + warn!("permanent error from logs intake: {status}; dropping batch"); + return Ok(()); + } + + // Transient server errors — fall through to retry. + warn!( + "transient error from logs intake: {status} (attempt {attempts}/{MAX_FLUSH_ATTEMPTS})" + ); + } + Err(e) => { + warn!( + "network error sending log batch (attempt {attempts}/{MAX_FLUSH_ATTEMPTS}): {e}" + ); + } + } + + if attempts >= MAX_FLUSH_ATTEMPTS { + warn!("log batch failed after {attempts} attempts; will retry next flush"); + return Err(builder); + } + } + } +} + +fn compress_zstd(data: &[u8], level: i32) -> Result, FlushError> { + let mut encoder = + Encoder::new(Vec::new(), level).map_err(|e| FlushError::Compression(e.to_string()))?; + encoder + .write_all(data) + .map_err(|e| FlushError::Compression(e.to_string()))?; + encoder + .finish() + .map_err(|e| FlushError::Compression(e.to_string())) +} + +#[cfg(test)] +// Tests use plain reqwest client to connect to local mock server +#[allow(clippy::disallowed_methods)] +mod tests { + use super::*; + use crate::aggregator::AggregatorService; + use crate::config::{Destination, LogFlusherConfig}; + use crate::intake_entry::IntakeEntry; + use crate::logs_additional_endpoint::LogsAdditionalEndpoint; + use mockito::Matcher; + use std::time::Duration; + + fn make_entry(msg: &str) -> IntakeEntry { + IntakeEntry::from_message(msg, 1_700_000_000_000) + } + + fn config_for_mock(mock_url: &str) -> LogFlusherConfig { + // Use OPW mode pointing at the mock server to avoid HTTPS + LogFlusherConfig { + api_key: "test-api-key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{mock_url}/api/v2/logs"), + }, + additional_endpoints: Vec::new(), + use_compression: false, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + } + } + + #[tokio::test] + async fn test_flush_empty_aggregator_does_not_call_api() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + // Server with no routes — any request would cause test failure + let mock_server = mockito::Server::new_async().await; + let config = config_for_mock(&mock_server.url()); + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle); + + assert!( + flusher.flush(vec![]).await.is_empty(), + "empty flush should succeed" + ); + // No mock assertions needed — absence of request is the assertion + } + + #[tokio::test] + async fn test_flush_sends_post_with_api_key_header() { + // Verify that Datadog mode sends both DD-API-KEY and DD-PROTOCOL: + // agent-json headers. We call ship_batch directly to bypass + // resolve_endpoint (which builds an HTTPS URL incompatible with the + // HTTP mock server). + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut mock_server = mockito::Server::new_async().await; + let mock = mock_server + .mock("POST", "/api/v2/logs") + .match_header("DD-API-KEY", "test-api-key") + .match_header("DD-PROTOCOL", "agent-json") + .with_status(202) + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "test-api-key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::Datadog, + additional_endpoints: Vec::new(), + use_compression: false, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle); + + // Call ship_batch directly to use the mock server's HTTP URL instead + // of the HTTPS URL that resolve_endpoint would produce. + let url = format!("{}/api/v2/logs", mock_server.url()); + let batch = b"[{\"message\":\"test\"}]"; + flusher + .ship_batch(batch, &url, false, "test-api-key", true) + .await + .expect("ship_batch should succeed"); + + mock.assert_async().await; + } + + #[tokio::test] + async fn test_flush_opw_mode_omits_dd_protocol_header() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut mock_server = mockito::Server::new_async().await; + let opw_url = format!("{}/logs", mock_server.url()); + + // Verify DD-PROTOCOL is NOT present in OPW requests + let mock = mock_server + .mock("POST", "/logs") + .match_header("DD-API-KEY", "test-api-key") + .match_header("DD-PROTOCOL", Matcher::Missing) + .with_status(200) + .expect(1) + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "test-api-key".to_string(), + site: "unused".to_string(), + mode: Destination::ObservabilityPipelinesWorker { url: opw_url }, + additional_endpoints: Vec::new(), + use_compression: false, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + + handle + .insert_batch(vec![make_entry("opw log")]) + .expect("insert"); + let result = flusher.flush(vec![]).await; + assert!(result.is_empty(), "OPW flush should return empty on 200"); + mock.assert_async().await; + } + + /// When the primary destination is OPW, additional endpoints are still Datadog + /// intakes and must receive the `DD-PROTOCOL: agent-json` header. + #[tokio::test] + async fn test_opw_primary_additional_endpoint_receives_dd_protocol_header() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut primary = mockito::Server::new_async().await; + let mut extra = mockito::Server::new_async().await; + + // OPW primary must NOT have DD-PROTOCOL header + let _primary_mock = primary + .mock("POST", "/logs") + .match_header("DD-PROTOCOL", Matcher::Missing) + .with_status(200) + .expect(1) + .create_async() + .await; + + // Additional endpoint (Datadog intake) MUST have DD-PROTOCOL header + let extra_mock = extra + .mock("POST", "/extra") + .match_header("DD-PROTOCOL", "agent-json") + .with_status(202) + .expect(1) + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/logs", primary.url()), + }, + additional_endpoints: vec![LogsAdditionalEndpoint { + api_key: "extra-key".to_string(), + url: format!("{}/extra", extra.url()), + is_reliable: true, + }], + use_compression: false, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + handle + .insert_batch(vec![make_entry("test")]) + .expect("insert"); + + assert!(flusher.flush(vec![]).await.is_empty()); + extra_mock.assert_async().await; + } + + #[tokio::test] + async fn test_flush_does_not_retry_on_403() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut mock_server = mockito::Server::new_async().await; + // expect(1) means exactly one call — if retried, the test will fail + let mock = mock_server + .mock("POST", "/api/v2/logs") + .with_status(403) + .expect(1) + .create_async() + .await; + + let config = config_for_mock(&mock_server.url()); + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + + handle + .insert_batch(vec![make_entry("log")]) + .expect("insert"); + let result = flusher.flush(vec![]).await; + // 403 is a permanent error — the batch is dropped, no builder to retry. + assert!( + result.is_empty(), + "403 is a permanent error; no builder to retry" + ); + mock.assert_async().await; + } + + /// 429 (Too Many Requests) is a retryable 4xx — the retry loop must + /// continue rather than short-circuiting with a permanent failure. + #[tokio::test] + async fn test_flush_retries_on_429_then_succeeds() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut mock_server = mockito::Server::new_async().await; + // First call → 429, second call → 200 + let _throttled = mock_server + .mock("POST", "/api/v2/logs") + .with_status(429) + .expect(1) + .create_async() + .await; + let _ok = mock_server + .mock("POST", "/api/v2/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + + let config = config_for_mock(&mock_server.url()); + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + + handle + .insert_batch(vec![make_entry("throttled log")]) + .expect("insert"); + let result = flusher.flush(vec![]).await; + assert!(result.is_empty(), "should succeed after 429 retry"); + } + + #[tokio::test] + async fn test_flush_retries_on_5xx_then_succeeds() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut mock_server = mockito::Server::new_async().await; + // First call → 500, second call → 202 + let _fail_mock = mock_server + .mock("POST", "/api/v2/logs") + .with_status(500) + .expect(1) + .create_async() + .await; + let _ok_mock = mock_server + .mock("POST", "/api/v2/logs") + .with_status(202) + .expect(1) + .create_async() + .await; + + let config = config_for_mock(&mock_server.url()); + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + + handle + .insert_batch(vec![make_entry("log")]) + .expect("insert"); + let result = flusher.flush(vec![]).await; + assert!(result.is_empty(), "should succeed on second attempt"); + } + + /// All additional endpoints receive the same batch when flush() is called. + #[tokio::test] + async fn test_additional_endpoints_all_receive_batch() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut primary = mockito::Server::new_async().await; + let mut extra1 = mockito::Server::new_async().await; + let mut extra2 = mockito::Server::new_async().await; + + let primary_mock = primary + .mock("POST", "/api/v2/logs") + .with_status(202) + .expect(1) + .create_async() + .await; + let extra1_mock = extra1 + .mock("POST", "/extra") + .with_status(200) + .expect(1) + .create_async() + .await; + let extra2_mock = extra2 + .mock("POST", "/extra") + .with_status(200) + .expect(1) + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/api/v2/logs", primary.url()), + }, + additional_endpoints: vec![ + LogsAdditionalEndpoint { + api_key: "extra-key-1".to_string(), + url: format!("{}/extra", extra1.url()), + is_reliable: true, + }, + LogsAdditionalEndpoint { + api_key: "extra-key-2".to_string(), + url: format!("{}/extra", extra2.url()), + is_reliable: true, + }, + ], + use_compression: false, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + handle.insert_batch(vec![make_entry("hi")]).expect("insert"); + + assert!(flusher.flush(vec![]).await.is_empty()); + primary_mock.assert_async().await; + extra1_mock.assert_async().await; + extra2_mock.assert_async().await; + } + + /// Additional endpoints are dispatched concurrently: if they were sequential, + /// two endpoints each waiting at a Barrier(2) would deadlock — only concurrent + /// dispatch lets both handlers reach the barrier simultaneously. + #[tokio::test] + async fn test_additional_endpoints_dispatched_concurrently() { + use std::sync::Arc; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpListener; + use tokio::sync::Barrier; + + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let barrier = Arc::new(Barrier::new(2)); + + // Spawn a minimal HTTP server that waits at the barrier before + // responding, so both must be in-flight at the same time to complete. + async fn serve_once(barrier: Arc) -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut buf = vec![0u8; 4096]; + let _ = stream.read(&mut buf).await; + barrier.wait().await; + let _ = stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await; + }); + format!("http://127.0.0.1:{}/logs", addr.port()) + } + + let url1 = serve_once(barrier.clone()).await; + let url2 = serve_once(barrier.clone()).await; + + let mut primary = mockito::Server::new_async().await; + let _primary_mock = primary + .mock("POST", "/api/v2/logs") + .with_status(202) + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/api/v2/logs", primary.url()), + }, + additional_endpoints: vec![ + LogsAdditionalEndpoint { + api_key: "extra-key-1".to_string(), + url: url1, + is_reliable: true, + }, + LogsAdditionalEndpoint { + api_key: "extra-key-2".to_string(), + url: url2, + is_reliable: true, + }, + ], + use_compression: false, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + handle + .insert_batch(vec![make_entry("concurrent")]) + .expect("insert"); + + assert!(flusher.flush(vec![]).await.is_empty()); + } + + /// A builder returned by `flush` can be redriven on the next call. + /// + /// The mock fails on the first 3 attempts (exhausting the per-invocation + /// retry budget), then succeeds on the 4th attempt (the next invocation). + /// This proves the cross-invocation retry path end-to-end. + #[tokio::test] + async fn test_cross_invocation_retry_delivers_on_redrive() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut mock_server = mockito::Server::new_async().await; + // First 3 calls: transient 503 → exhausts per-invocation retry budget + let _fail_mock = mock_server + .mock("POST", "/api/v2/logs") + .with_status(503) + .expect(3) + .create_async() + .await; + // 4th call: redriven on the next flush → succeeds + let _ok_mock = mock_server + .mock("POST", "/api/v2/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + + let config = config_for_mock(&mock_server.url()); + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + handle + .insert_batch(vec![make_entry("retry-me")]) + .expect("insert"); + + // First flush: all 3 attempts fail → returns the builder for retry. + let failed = flusher.flush(vec![]).await; + assert_eq!(failed.len(), 1, "one builder should be returned for retry"); + + // Second flush: aggregator is empty; redrives the failed builder → succeeds. + let result = flusher.flush(failed).await; + assert!( + result.is_empty(), + "redriven builder should succeed on the next invocation" + ); + } + + /// When the primary is OPW (compression disabled for OPW transport) and + /// `use_compression` is true, additional Datadog endpoints must still + /// receive `Content-Encoding: zstd` — they are Datadog intakes and honour + /// `DD_LOGS_CONFIG_USE_COMPRESSION` independently of the primary. + #[tokio::test] + async fn test_opw_primary_additional_endpoint_compresses_when_enabled() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut primary = mockito::Server::new_async().await; + let mut extra = mockito::Server::new_async().await; + + // OPW primary must NOT have Content-Encoding + let _primary_mock = primary + .mock("POST", "/logs") + .match_header("Content-Encoding", Matcher::Missing) + .with_status(200) + .expect(1) + .create_async() + .await; + + // Additional Datadog endpoint MUST receive Content-Encoding: zstd + let extra_mock = extra + .mock("POST", "/extra") + .match_header("Content-Encoding", "zstd") + .with_status(202) + .expect(1) + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/logs", primary.url()), + }, + additional_endpoints: vec![LogsAdditionalEndpoint { + api_key: "extra-key".to_string(), + url: format!("{}/extra", extra.url()), + is_reliable: true, + }], + use_compression: true, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + handle + .insert_batch(vec![make_entry("compressed extra")]) + .expect("insert"); + + assert!(flusher.flush(vec![]).await.is_empty()); + extra_mock.assert_async().await; + } + + /// Even when `use_compression: true`, the OPW primary endpoint must never + /// receive a `Content-Encoding` header — OPW does not support zstd. + #[tokio::test] + async fn test_opw_primary_never_compressed_even_when_flag_set() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut mock_server = mockito::Server::new_async().await; + let mock = mock_server + .mock("POST", "/logs") + .match_header("Content-Encoding", Matcher::Missing) + .with_status(200) + .expect(1) + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/logs", mock_server.url()), + }, + additional_endpoints: vec![], + use_compression: true, // flag set but must not reach OPW + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + handle + .insert_batch(vec![make_entry("opw no compress")]) + .expect("insert"); + + assert!(flusher.flush(vec![]).await.is_empty()); + mock.assert_async().await; + } + + /// Additional-endpoint failures are best-effort: their builders are NOT + /// included in the returned retry vec, even when they exhaust all retries. + /// Only primary-endpoint failures are tracked for cross-invocation retry. + #[tokio::test] + async fn test_additional_endpoint_failures_not_tracked_for_retry() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut primary = mockito::Server::new_async().await; + let mut extra = mockito::Server::new_async().await; + + let _primary_mock = primary + .mock("POST", "/api/v2/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + // Additional endpoint always returns 503 — exhausts per-invocation retries. + let _extra_mock = extra + .mock("POST", "/extra") + .with_status(503) + .expect(3) // MAX_FLUSH_ATTEMPTS + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/api/v2/logs", primary.url()), + }, + additional_endpoints: vec![LogsAdditionalEndpoint { + api_key: "extra-key".to_string(), + url: format!("{}/extra", extra.url()), + is_reliable: true, + }], + use_compression: false, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + handle + .insert_batch(vec![make_entry("test")]) + .expect("insert"); + + let result = flusher.flush(vec![]).await; + assert!( + result.is_empty(), + "additional-endpoint failures are best-effort and must not be tracked for retry" + ); + } +} diff --git a/crates/datadog-logs-agent/src/intake_entry.rs b/crates/datadog-logs-agent/src/intake_entry.rs new file mode 100644 index 0000000..2ec0186 --- /dev/null +++ b/crates/datadog-logs-agent/src/intake_entry.rs @@ -0,0 +1,188 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use serde::{Deserialize, Serialize}; + +/// A single log entry in the Datadog Logs intake format. +/// +/// Standard Datadog fields are typed fields. Runtime-specific enrichment +/// (e.g. `{"lambda": {"arn": "...", "request_id": "..."}}` for Lambda, +/// `{"azure": {"resource_id": "..."}}` for Azure Functions) goes in `attributes`, +/// which is flattened into the JSON object at serialization time. +/// +/// # Example — Lambda extension consumer +/// ```ignore +/// let mut attrs = serde_json::Map::new(); +/// attrs.insert("lambda".to_string(), serde_json::json!({ +/// "arn": function_arn, +/// "request_id": request_id, +/// })); +/// let entry = IntakeEntry { +/// message: log_line, +/// timestamp: timestamp_ms, +/// hostname: Some(function_arn), +/// service: Some(service_name), +/// ddsource: Some("lambda".to_string()), +/// ddtags: Some(tags), +/// status: Some("info".to_string()), +/// attributes: attrs, +/// }; +/// ``` +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IntakeEntry { + /// The log message body. + pub message: String, + + /// Unix timestamp in milliseconds. + pub timestamp: i64, + + /// The hostname (e.g. Lambda function ARN, Azure resource ID). + #[serde(skip_serializing_if = "Option::is_none")] + pub hostname: Option, + + /// The service name. + #[serde(skip_serializing_if = "Option::is_none")] + pub service: Option, + + /// The log source tag (e.g. "lambda", "azure-functions", "gcp-functions"). + #[serde(skip_serializing_if = "Option::is_none")] + pub ddsource: Option, + + /// Comma-separated Datadog tags (e.g. "env:prod,version:1.0"). + #[serde(skip_serializing_if = "Option::is_none")] + pub ddtags: Option, + + /// Log level / status (e.g. "info", "error", "warn", "debug"). + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, + + /// Runtime-specific enrichment fields, flattened into the JSON object. + /// Use this for fields like `{"lambda": {"arn": "...", "request_id": "..."}}`. + /// An empty map is not serialized. Extra fields in JSON are collected here on deserialization. + #[serde(flatten, default, skip_serializing_if = "serde_json::Map::is_empty")] + pub attributes: serde_json::Map, +} + +impl IntakeEntry { + /// Create a minimal log entry from a message and timestamp. + /// All optional fields default to `None`; use struct literal syntax to set them. + pub fn from_message(message: impl Into, timestamp: i64) -> Self { + Self { + message: message.into(), + timestamp, + hostname: None, + service: None, + ddsource: None, + ddtags: None, + status: None, + attributes: serde_json::Map::new(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_intake_entry_minimal_serialization() { + let entry = IntakeEntry::from_message("hello world", 1_700_000_000_000); + let json = serde_json::to_string(&entry).expect("serialize"); + let v: serde_json::Value = serde_json::from_str(&json).expect("parse"); + + assert_eq!(v["message"], "hello world"); + assert_eq!(v["timestamp"], 1_700_000_000_000_i64); + // Optional fields must be absent when None + assert!(v.get("hostname").is_none()); + assert!(v.get("service").is_none()); + assert!(v.get("ddsource").is_none()); + assert!(v.get("ddtags").is_none()); + assert!(v.get("status").is_none()); + } + + #[test] + fn test_intake_entry_full_serialization() { + let entry = IntakeEntry { + message: "user logged in".to_string(), + timestamp: 1_700_000_001_000, + hostname: Some("my-host".to_string()), + service: Some("my-service".to_string()), + ddsource: Some("lambda".to_string()), + ddtags: Some("env:prod,version:1.0".to_string()), + status: Some("info".to_string()), + attributes: serde_json::Map::new(), + }; + let json = serde_json::to_string(&entry).expect("serialize"); + let v: serde_json::Value = serde_json::from_str(&json).expect("parse"); + + assert_eq!(v["message"], "user logged in"); + assert_eq!(v["hostname"], "my-host"); + assert_eq!(v["service"], "my-service"); + assert_eq!(v["ddsource"], "lambda"); + assert_eq!(v["ddtags"], "env:prod,version:1.0"); + assert_eq!(v["status"], "info"); + assert!( + v.get("attributes").is_none(), + "empty attributes must not appear in output" + ); + } + + #[test] + fn test_intake_entry_with_lambda_attributes_flattened() { + // Simulates what the lambda extension would build + let mut attrs = serde_json::Map::new(); + attrs.insert( + "lambda".to_string(), + serde_json::json!({ + "arn": "arn:aws:lambda:us-east-1:123456789012:function:my-fn", + "request_id": "abc-123" + }), + ); + let entry = IntakeEntry { + message: "function invoked".to_string(), + timestamp: 1_700_000_002_000, + hostname: Some("arn:aws:lambda:us-east-1:123456789012:function:my-fn".to_string()), + service: Some("my-fn".to_string()), + ddsource: Some("lambda".to_string()), + ddtags: Some("env:prod".to_string()), + status: Some("info".to_string()), + attributes: attrs, + }; + let json = serde_json::to_string(&entry).expect("serialize"); + let v: serde_json::Value = serde_json::from_str(&json).expect("parse"); + + // Lambda-specific fields appear at top level (flattened) + assert_eq!( + v["lambda"]["arn"], + "arn:aws:lambda:us-east-1:123456789012:function:my-fn" + ); + assert_eq!(v["lambda"]["request_id"], "abc-123"); + assert_eq!(v["message"], "function invoked"); + } + + #[test] + fn test_intake_entry_deserialization_roundtrip() { + let original = IntakeEntry { + message: "test".to_string(), + timestamp: 42, + hostname: Some("h".to_string()), + service: None, + ddsource: Some("gcp-functions".to_string()), + ddtags: None, + status: Some("error".to_string()), + attributes: serde_json::Map::new(), + }; + let json = serde_json::to_string(&original).expect("serialize"); + let restored: IntakeEntry = serde_json::from_str(&json).expect("deserialize"); + + assert_eq!(restored.message, original.message); + assert_eq!(restored.timestamp, original.timestamp); + assert_eq!(restored.hostname, original.hostname); + assert_eq!(restored.ddsource, original.ddsource); + assert_eq!(restored.status, original.status); + assert!( + restored.attributes.is_empty(), + "no extra attributes expected after roundtrip" + ); + } +} diff --git a/crates/datadog-logs-agent/src/lib.rs b/crates/datadog-logs-agent/src/lib.rs new file mode 100644 index 0000000..ef1c454 --- /dev/null +++ b/crates/datadog-logs-agent/src/lib.rs @@ -0,0 +1,26 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +#![cfg_attr(not(test), deny(clippy::panic))] +#![cfg_attr(not(test), deny(clippy::unwrap_used))] +#![cfg_attr(not(test), deny(clippy::expect_used))] +#![cfg_attr(not(test), deny(clippy::todo))] +#![cfg_attr(not(test), deny(clippy::unimplemented))] + +pub mod aggregator; +pub mod config; +pub mod constants; +pub mod errors; +pub mod flusher; +pub mod intake_entry; +pub mod logs_additional_endpoint; + +pub mod server; + +// Re-export the most commonly used types at the crate root +pub use aggregator::{AggregatorHandle, AggregatorService}; +pub use config::{Destination, LogFlusherConfig}; +pub use flusher::LogFlusher; +pub use intake_entry::IntakeEntry; +pub use logs_additional_endpoint::LogsAdditionalEndpoint; +pub use server::{LogServer, LogServerConfig}; diff --git a/crates/datadog-logs-agent/src/logs_additional_endpoint.rs b/crates/datadog-logs-agent/src/logs_additional_endpoint.rs new file mode 100644 index 0000000..9413e0b --- /dev/null +++ b/crates/datadog-logs-agent/src/logs_additional_endpoint.rs @@ -0,0 +1,104 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use serde::Deserialize; +use tracing::warn; + +/// An additional Datadog intake endpoint to ship each log batch to alongside +/// the primary endpoint. +/// +/// The JSON wire format (from `DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS`) matches the +/// bottlecap / datadog-agent convention: +/// ```json +/// [{"api_key":"","Host":"agent-http-intake.logs.datadoghq.com","Port":443,"is_reliable":true}] +/// ``` +#[derive(Debug, PartialEq, Clone)] +pub struct LogsAdditionalEndpoint { + /// API key used exclusively for this endpoint. + pub api_key: String, + /// Full intake URL, e.g. `https://agent-http-intake.logs.datadoghq.com:443/api/v2/logs`. + /// Computed from `Host` and `Port` at deserialize time. + pub url: String, + /// When `true`, failures on this endpoint are counted toward overall flush reliability. + /// Currently stored but not yet acted upon; reserved for future use. + pub is_reliable: bool, +} + +/// Internal representation that mirrors the JSON wire format. +#[derive(Deserialize)] +struct RawEndpoint { + api_key: String, + #[serde(rename = "Host")] + host: String, + #[serde(rename = "Port")] + port: u32, + is_reliable: bool, +} + +impl From for LogsAdditionalEndpoint { + fn from(r: RawEndpoint) -> Self { + Self { + api_key: r.api_key, + url: format!("https://{}:{}/api/v2/logs", r.host, r.port), + is_reliable: r.is_reliable, + } + } +} + +/// Parse the value of `DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS` (a JSON array string). +/// +/// Returns an empty `Vec` and emits a warning on parse failure. +pub fn parse_additional_endpoints(s: &str) -> Vec { + match serde_json::from_str::>(s) { + Ok(raw) => raw.into_iter().map(Into::into).collect(), + Err(e) => { + warn!("failed to parse DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS: {e}"); + vec![] + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_valid_endpoint() { + let s = r#"[{"api_key":"key2","Host":"agent-http-intake.logs.datadoghq.com","Port":443,"is_reliable":true}]"#; + let result = parse_additional_endpoints(s); + assert_eq!(result.len(), 1); + assert_eq!(result[0].api_key, "key2"); + assert_eq!( + result[0].url, + "https://agent-http-intake.logs.datadoghq.com:443/api/v2/logs" + ); + assert!(result[0].is_reliable); + } + + #[test] + fn test_parse_missing_port_returns_empty() { + // Missing required "Port" field — should warn and return [] + let s = r#"[{"api_key":"key","Host":"intake.logs.datadoghq.com","is_reliable":true}]"#; + let result = parse_additional_endpoints(s); + assert!(result.is_empty()); + } + + #[test] + fn test_parse_empty_string_returns_empty() { + let result = parse_additional_endpoints(""); + assert!(result.is_empty()); + } + + #[test] + fn test_parse_multiple_endpoints() { + let s = r#"[ + {"api_key":"k1","Host":"host1.example.com","Port":443,"is_reliable":true}, + {"api_key":"k2","Host":"host2.example.com","Port":10516,"is_reliable":false} + ]"#; + let result = parse_additional_endpoints(s); + assert_eq!(result.len(), 2); + assert_eq!(result[0].url, "https://host1.example.com:443/api/v2/logs"); + assert_eq!(result[1].url, "https://host2.example.com:10516/api/v2/logs"); + assert!(!result[1].is_reliable); + } +} diff --git a/crates/datadog-logs-agent/src/server.rs b/crates/datadog-logs-agent/src/server.rs new file mode 100644 index 0000000..bc7981a --- /dev/null +++ b/crates/datadog-logs-agent/src/server.rs @@ -0,0 +1,523 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! HTTP intake server for the log agent. +//! +//! [`LogServer`] listens on a TCP port and accepts `POST /v1/input` requests +//! whose body is a JSON array of [`crate::IntakeEntry`] values. Entries are +//! forwarded to the shared [`crate::AggregatorHandle`] for batching and +//! eventual flushing. +//! +//! # Usage (network intake — serverless-compat) +//! ```ignore +//! let (service, handle) = AggregatorService::new(); +//! tokio::spawn(service.run()); +//! +//! let server = LogServer::new( +//! LogServerConfig { host: "0.0.0.0".into(), port: 8080 }, +//! handle.clone(), +//! ); +//! tokio::spawn(server.serve()); +//! +//! let flusher = LogFlusher::new(config, client, handle); +//! // flush periodically … +//! ``` +//! +//! # Direct intake (bottlecap — unchanged) +//! ```ignore +//! // bottlecap never uses LogServer; it calls handle.insert_batch() directly. +//! handle.insert_batch(entries).expect("insert"); +//! ``` + +use http_body_util::BodyExt as _; +use hyper::body::Incoming; +use hyper::service::service_fn; +use hyper::{Method, Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use tracing::{debug, error, warn}; + +use crate::aggregator::AggregatorHandle; +use crate::intake_entry::IntakeEntry; + +const LOG_INTAKE_PATH: &str = "/v1/input"; +/// Maximum accepted request body size in bytes (4 MiB). Requests larger than +/// this are rejected with 413 before the body is read into memory. +const MAX_BODY_BYTES: usize = 4 * 1024 * 1024; + +/// Configuration for the [`LogServer`] HTTP intake listener. +#[derive(Debug, Clone)] +pub struct LogServerConfig { + /// Interface to bind (e.g. `"0.0.0.0"` or `"127.0.0.1"`). + pub host: String, + /// TCP port to listen on. + pub port: u16, +} + +/// HTTP server that receives log entries over the network and forwards them to +/// a running [`AggregatorHandle`]. +/// +/// Create with [`LogServer::new`], then call [`LogServer::serve`] inside a +/// `tokio::spawn` — it runs forever until the process exits. +pub struct LogServer { + config: LogServerConfig, + handle: AggregatorHandle, +} + +impl LogServer { + /// Create a new server. Does **not** bind the port until [`serve`](Self::serve) is called. + pub fn new(config: LogServerConfig, handle: AggregatorHandle) -> Self { + Self { config, handle } + } + + /// Bind the configured port and serve HTTP/1 requests indefinitely. + /// + /// This is an `async fn` meant to be run inside `tokio::spawn`. + /// It only returns if binding fails; otherwise it loops forever. + pub async fn serve(self) { + let addr = format!("{}:{}", self.config.host, self.config.port); + let listener = match tokio::net::TcpListener::bind(&addr).await { + Ok(l) => { + let actual = l.local_addr().map_or(addr.clone(), |a| a.to_string()); + debug!("log server listening on {actual}"); + l + } + Err(e) => { + error!("log server failed to bind {addr}: {e}"); + return; + } + }; + + loop { + let (stream, peer) = match listener.accept().await { + Ok(pair) => pair, + Err(e) => { + warn!("log server accept error: {e}"); + continue; + } + }; + + debug!("log server: connection from {peer}"); + let handle = self.handle.clone(); + tokio::spawn(async move { + let io = TokioIo::new(stream); + let svc = service_fn(move |req: Request| { + let handle = handle.clone(); + async move { handle_request(req, handle).await } + }); + if let Err(e) = hyper::server::conn::http1::Builder::new() + .serve_connection(io, svc) + .await + { + debug!("log server: connection error: {e}"); + } + }); + } + } +} + +/// Handle a single HTTP request: route, parse body, insert into aggregator. +async fn handle_request( + req: Request, + handle: AggregatorHandle, +) -> Result, std::convert::Infallible> { + if req.method() != Method::POST { + return Ok(Response::builder() + .status(StatusCode::METHOD_NOT_ALLOWED) + .body("method not allowed".to_string()) + .unwrap_or_default()); + } + if req.uri().path() != LOG_INTAKE_PATH { + return Ok(Response::builder() + .status(StatusCode::NOT_FOUND) + .body("not found".to_string()) + .unwrap_or_default()); + } + + // Reject early if Content-Length is declared and already exceeds the limit. + // Skip this check when Content-Length is absent (chunked transfer) — actual + // size is enforced after reading below. + if let Some(content_length) = req + .headers() + .get(hyper::header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()) + && content_length > MAX_BODY_BYTES + { + return Ok(Response::builder() + .status(StatusCode::PAYLOAD_TOO_LARGE) + .body("payload too large".to_string()) + .unwrap_or_default()); + } + + // TODO(SVLS-chunked-body-limit): the 4 MiB cap is only enforced *after* + // `collect()` has buffered the entire body. For chunked requests (no + // Content-Length) an attacker can stream an arbitrarily large body before + // we ever reject with 413. + // Fix: replace `req.collect()` with + // `Limited::new(req.into_body(), MAX_BODY_BYTES).collect()` from + // `http_body_util` (already a dependency). `Limited` aborts mid-stream as + // soon as the threshold is exceeded. Match on `LengthLimitError` via + // `e.downcast_ref::().is_some()` for the 413 branch, and + // remove the now-redundant post-read `bytes.len() > MAX_BODY_BYTES` check. + // Add test `test_chunked_oversized_body_returns_413`: send MAX_BODY_BYTES+1 + // bytes over raw chunked TCP using `TcpStream::into_split()` for concurrent + // write/read (needed to avoid deadlock when the OS socket buffer fills up). + let bytes = match req.collect().await { + Ok(collected) => collected.to_bytes(), + Err(e) => { + warn!("log server: failed to read request body: {e}"); + return Ok(Response::builder() + .status(StatusCode::BAD_REQUEST) + .body("failed to read body".to_string()) + .unwrap_or_default()); + } + }; + if bytes.len() > MAX_BODY_BYTES { + return Ok(Response::builder() + .status(StatusCode::PAYLOAD_TOO_LARGE) + .body("payload too large".to_string()) + .unwrap_or_default()); + } + + let entries: Vec = match serde_json::from_slice(&bytes) { + Ok(e) => e, + Err(e) => { + warn!("log server: failed to parse log entries: {e}"); + return Ok(Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(format!("invalid JSON: {e}")) + .unwrap_or_default()); + } + }; + + if entries.is_empty() { + return Ok(Response::builder() + .status(StatusCode::OK) + .body("ok".to_string()) + .unwrap_or_default()); + } + + debug!("log server: received {} entries", entries.len()); + + if let Err(e) = handle.insert_batch(entries) { + error!("log server: failed to insert batch: {e}"); + return Ok(Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body("aggregator unavailable".to_string()) + .unwrap_or_default()); + } + + Ok(Response::builder() + .status(StatusCode::OK) + .body("ok".to_string()) + .unwrap_or_default()) +} + +#[cfg(test)] +// Tests use plain reqwest; FIPS client not needed for local loopback +#[allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)] +mod tests { + use super::*; + use crate::aggregator::AggregatorService; + use tokio::time::{Duration, sleep}; + + /// Bind `:0`, record the OS-assigned port, drop the listener, then start + /// `LogServer` on that port. Returns the base URL. + async fn start_test_server(handle: AggregatorHandle) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + + let server = LogServer::new( + LogServerConfig { + host: "127.0.0.1".into(), + port, + }, + handle, + ); + tokio::spawn(server.serve()); + sleep(Duration::from_millis(50)).await; + format!("http://127.0.0.1:{port}") + } + + #[tokio::test] + async fn test_post_valid_entries_returns_200_and_batch_inserted() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle.clone()).await; + let client = reqwest::Client::new(); + + let entries = vec![ + serde_json::json!({"message": "hello", "timestamp": 1_700_000_000_000_i64}), + serde_json::json!({"message": "world", "timestamp": 1_700_000_001_000_i64}), + ]; + + let resp = client + .post(format!("{base_url}/v1/input")) + .json(&entries) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), 200); + + let batches = handle.get_batches().await.expect("get_batches"); + assert_eq!(batches.len(), 1, "should have one batch"); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).unwrap(); + assert_eq!(arr.as_array().unwrap().len(), 2); + assert_eq!(arr[0]["message"], "hello"); + } + + #[tokio::test] + async fn test_post_malformed_json_returns_400() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle).await; + let client = reqwest::Client::new(); + + let resp = client + .post(format!("{base_url}/v1/input")) + .header("Content-Type", "application/json") + .body("not-json") + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), 400); + } + + #[tokio::test] + async fn test_get_request_returns_405() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle).await; + let client = reqwest::Client::new(); + + let resp = client + .get(format!("{base_url}/v1/input")) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), 405); + } + + #[tokio::test] + async fn test_wrong_path_returns_404() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle).await; + let client = reqwest::Client::new(); + + let resp = client + .post(format!("{base_url}/wrong/path")) + .json(&serde_json::json!([{"message": "x", "timestamp": 0_i64}])) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), 404); + } + + #[tokio::test] + async fn test_post_empty_array_returns_200_no_batch() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle.clone()).await; + let client = reqwest::Client::new(); + + let resp = client + .post(format!("{base_url}/v1/input")) + .json(&serde_json::json!([])) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), 200); + + let batches = handle.get_batches().await.expect("get_batches"); + assert!(batches.is_empty(), "empty POST should insert nothing"); + } + + /// A request whose Content-Length header exceeds MAX_BODY_BYTES must be + /// rejected with 413 before any body bytes are read. + #[tokio::test] + async fn test_oversized_content_length_returns_413() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle).await; + let client = reqwest::Client::new(); + + // The server checks the Content-Length header directly and rejects + // before reading the body when the declared size exceeds the limit. + let fake_large_size = MAX_BODY_BYTES + 1; + let resp = client + .post(format!("{base_url}/v1/input")) + .header("Content-Type", "application/json") + .header("Content-Length", fake_large_size.to_string()) + .body("[]") + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), 413); + } + + /// A POST with Transfer-Encoding: chunked (no Content-Length header) must + /// not be rejected with 413. This is the regression test for the original + /// bug where `size_hint().upper()` returning `None` was coerced to `u64::MAX` + /// and treated as exceeding the body-size limit. + #[tokio::test] + async fn test_chunked_transfer_encoding_accepted() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpStream; + + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle.clone()).await; + let port: u16 = base_url + .trim_start_matches("http://127.0.0.1:") + .parse() + .expect("port"); + + let body = r#"[{"message":"chunked","timestamp":1700000000000}]"#; + let request = format!( + "POST /v1/input HTTP/1.1\r\n\ + Host: 127.0.0.1:{port}\r\n\ + Content-Type: application/json\r\n\ + Transfer-Encoding: chunked\r\n\ + \r\n\ + {:x}\r\n\ + {body}\r\n\ + 0\r\n\ + \r\n", + body.len(), + ); + + let mut stream = TcpStream::connect(format!("127.0.0.1:{port}")) + .await + .expect("connect"); + stream.write_all(request.as_bytes()).await.expect("write"); + stream.flush().await.expect("flush"); + + let mut response = String::new(); + let mut buf = [0u8; 4096]; + loop { + let n = stream.read(&mut buf).await.expect("read"); + if n == 0 { + break; + } + response.push_str(&String::from_utf8_lossy(&buf[..n])); + if response.contains("\r\n\r\n") { + break; + } + } + + assert!( + response.starts_with("HTTP/1.1 200"), + "expected 200, got: {response}" + ); + let batches = handle.get_batches().await.expect("get_batches"); + assert_eq!(batches.len(), 1, "entry should have been inserted"); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).unwrap(); + assert_eq!(arr[0]["message"], "chunked"); + } + + /// All optional IntakeEntry fields (hostname, service, ddsource, ddtags, + /// status) and arbitrary attributes must survive the HTTP round-trip + /// through the server and appear intact in the aggregated batch. + #[tokio::test] + async fn test_full_intake_entry_fields_preserved_through_http() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle.clone()).await; + let client = reqwest::Client::new(); + + let payload = serde_json::json!([{ + "message": "lambda invoked", + "timestamp": 1_700_000_002_000_i64, + "hostname": "arn:aws:lambda:us-east-1:123:function:my-fn", + "service": "my-fn", + "ddsource": "lambda", + "ddtags": "env:prod,version:1.0", + "status": "info", + "lambda": { + "arn": "arn:aws:lambda:us-east-1:123:function:my-fn", + "request_id": "req-abc-123" + } + }]); + + let resp = client + .post(format!("{base_url}/v1/input")) + .json(&payload) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), 200); + + let batches = handle.get_batches().await.expect("get_batches"); + assert_eq!(batches.len(), 1); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).unwrap(); + let entry = &arr[0]; + + assert_eq!(entry["message"], "lambda invoked"); + assert_eq!( + entry["hostname"], + "arn:aws:lambda:us-east-1:123:function:my-fn" + ); + assert_eq!(entry["service"], "my-fn"); + assert_eq!(entry["ddsource"], "lambda"); + assert_eq!(entry["ddtags"], "env:prod,version:1.0"); + assert_eq!(entry["status"], "info"); + // Flattened attributes must appear at the top level + assert_eq!(entry["lambda"]["request_id"], "req-abc-123"); + } + + /// Two sequential POST requests must both accumulate in the aggregator + /// before `get_batches` drains them. + #[tokio::test] + async fn test_sequential_posts_accumulate_in_aggregator() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle.clone()).await; + let client = reqwest::Client::new(); + + // First request + client + .post(format!("{base_url}/v1/input")) + .json(&serde_json::json!([{"message": "first", "timestamp": 1_i64}])) + .send() + .await + .expect("first request failed"); + + // Second request + client + .post(format!("{base_url}/v1/input")) + .json(&serde_json::json!([{"message": "second", "timestamp": 2_i64}])) + .send() + .await + .expect("second request failed"); + + let batches = handle.get_batches().await.expect("get_batches"); + // Both entries land in the same aggregator; batch count depends on + // the aggregator's internal sizing, but total entries must be 2. + let total_entries: usize = batches + .iter() + .map(|b| { + let arr: serde_json::Value = serde_json::from_slice(b).unwrap(); + arr.as_array().unwrap().len() + }) + .sum(); + assert_eq!(total_entries, 2, "both entries should be in the aggregator"); + } +} diff --git a/crates/datadog-logs-agent/tests/integration_test.rs b/crates/datadog-logs-agent/tests/integration_test.rs new file mode 100644 index 0000000..291777c --- /dev/null +++ b/crates/datadog-logs-agent/tests/integration_test.rs @@ -0,0 +1,846 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Integration tests for the `datadog-logs-agent` crate. +//! +//! These tests exercise two intake paths: +//! +//! **Direct intake** (bottlecap / in-process): +//! `IntakeEntry` → `AggregatorHandle::insert_batch` → `LogFlusher::flush` → HTTP endpoint +//! +//! **Network intake** (serverless-compat / over HTTP): +//! HTTP POST → `LogServer` → `AggregatorHandle::insert_batch` → `LogFlusher::flush` → HTTP endpoint +//! +//! HTTP traffic is directed to a local `mockito` server via +//! `Destination::ObservabilityPipelinesWorker`, which accepts a direct URL. +//! Datadog-mode-specific headers (`DD-PROTOCOL`) are covered by unit tests in `flusher.rs`. + +#![allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)] + +use datadog_logs_agent::{ + AggregatorService, Destination, IntakeEntry, LogFlusher, LogFlusherConfig, LogServer, + LogServerConfig, LogsAdditionalEndpoint, +}; +use mockito::{Matcher, Server}; +use std::time::Duration; +use tokio::time::sleep; + +// ── Helpers ────────────────────────────────────────────────────────────────── + +fn build_client() -> reqwest::Client { + reqwest::Client::builder() + .build() + .expect("failed to build HTTP client") +} + +/// Config that routes all flushes to `mock_url/logs` via OPW mode. +fn opw_config(mock_url: &str) -> LogFlusherConfig { + LogFlusherConfig { + api_key: "test-api-key".to_string(), + site: "ignored.datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/logs", mock_url), + }, + additional_endpoints: Vec::new(), + use_compression: false, + compression_level: 0, + flush_timeout: Duration::from_secs(5), + } +} + +fn entry(msg: &str) -> IntakeEntry { + IntakeEntry::from_message(msg, 1_700_000_000_000) +} + +// ── Pipeline happy path ─────────────────────────────────────────────────────── + +/// Inserting log entries and flushing sends a single POST to the endpoint. +#[tokio::test] +async fn test_pipeline_inserts_and_flushes() { + let mut server = Server::new_async().await; + let mock = server + .mock("POST", "/logs") + .match_header("DD-API-KEY", "test-api-key") + .with_status(200) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + handle + .insert_batch(vec![entry("hello"), entry("world")]) + .expect("insert_batch"); + + let result = LogFlusher::new(opw_config(&server.url()), build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty(), "flush should return empty on 200"); + mock.assert_async().await; +} + +/// Flushing with no entries makes no HTTP request. +#[tokio::test] +async fn test_empty_flush_makes_no_request() { + let server = Server::new_async().await; + // Any unexpected request would return 501 and cause an assertion failure below. + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let url = server.url(); + let result = LogFlusher::new(opw_config(&url), build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty(), "empty flush should return empty"); + // No mock was set up — if a request had been made, mockito would panic. + drop(server); +} + +// ── JSON payload shape ──────────────────────────────────────────────────────── + +/// The flushed payload is a valid JSON array containing each inserted entry. +#[tokio::test] +async fn test_payload_is_json_array_with_correct_fields() { + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + handle + .insert_batch(vec![IntakeEntry { + message: "user login".to_string(), + timestamp: 1_700_000_001_000, + hostname: Some("web-01".to_string()), + service: Some("auth".to_string()), + ddsource: Some("nodejs".to_string()), + ddtags: Some("env:prod,version:2.0".to_string()), + status: Some("info".to_string()), + attributes: serde_json::Map::new(), + }]) + .expect("insert"); + + let batches = handle.get_batches().await.expect("get_batches"); + assert_eq!(batches.len(), 1); + + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("valid JSON"); + let entries = arr.as_array().expect("JSON array"); + assert_eq!(entries.len(), 1); + + let e = &entries[0]; + assert_eq!(e["message"], "user login"); + assert_eq!(e["timestamp"], 1_700_000_001_000_i64); + assert_eq!(e["hostname"], "web-01"); + assert_eq!(e["service"], "auth"); + assert_eq!(e["ddsource"], "nodejs"); + assert_eq!(e["ddtags"], "env:prod,version:2.0"); + assert_eq!(e["status"], "info"); +} + +/// Absent optional fields are not serialized into the JSON payload. +#[tokio::test] +async fn test_absent_optional_fields_not_serialized() { + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + handle + .insert_batch(vec![IntakeEntry::from_message("minimal", 0)]) + .expect("insert"); + + let batches = handle.get_batches().await.expect("get_batches"); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("valid JSON"); + let e = &arr[0]; + + assert_eq!(e["message"], "minimal"); + assert!(e.get("hostname").is_none(), "hostname absent"); + assert!(e.get("service").is_none(), "service absent"); + assert!(e.get("ddsource").is_none(), "ddsource absent"); + assert!(e.get("ddtags").is_none(), "ddtags absent"); + assert!(e.get("status").is_none(), "status absent"); +} + +// ── Runtime-specific attributes ─────────────────────────────────────────────── + +/// Lambda-specific attributes are flattened into the top-level JSON object. +#[tokio::test] +async fn test_lambda_attributes_flattened_at_top_level() { + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let mut attrs = serde_json::Map::new(); + attrs.insert( + "lambda".to_string(), + serde_json::json!({ + "arn": "arn:aws:lambda:us-east-1:123456789012:function:my-fn", + "request_id": "abc-123" + }), + ); + + handle + .insert_batch(vec![IntakeEntry { + message: "invocation complete".to_string(), + timestamp: 0, + hostname: Some("my-fn".to_string()), + service: Some("my-fn".to_string()), + ddsource: Some("lambda".to_string()), + ddtags: Some("env:prod".to_string()), + status: Some("info".to_string()), + attributes: attrs, + }]) + .expect("insert"); + + let batches = handle.get_batches().await.expect("get_batches"); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("valid JSON"); + let e = &arr[0]; + + // Lambda object is a top-level key (flattened via #[serde(flatten)]) + assert_eq!( + e["lambda"]["arn"], + "arn:aws:lambda:us-east-1:123456789012:function:my-fn" + ); + assert_eq!(e["lambda"]["request_id"], "abc-123"); +} + +/// Azure-specific attributes are flattened into the top-level JSON object. +#[tokio::test] +async fn test_azure_attributes_flattened_at_top_level() { + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let mut attrs = serde_json::Map::new(); + attrs.insert( + "azure".to_string(), + serde_json::json!({ + "resource_id": "/subscriptions/sub-123/resourceGroups/rg/providers/Microsoft.Web/sites/my-fn", + "operation_name": "Microsoft.Web/sites/functions/run/action" + }), + ); + + handle + .insert_batch(vec![IntakeEntry { + message: "azure function triggered".to_string(), + timestamp: 0, + hostname: Some("my-azure-fn".to_string()), + service: Some("payments".to_string()), + ddsource: Some("azure-functions".to_string()), + ddtags: Some("env:staging".to_string()), + status: Some("info".to_string()), + attributes: attrs, + }]) + .expect("insert"); + + let batches = handle.get_batches().await.expect("get_batches"); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("valid JSON"); + let e = &arr[0]; + + assert_eq!(e["ddsource"], "azure-functions"); + assert!( + e["azure"]["resource_id"] + .as_str() + .unwrap_or("") + .contains("Microsoft.Web"), + "azure resource_id present" + ); + assert_eq!( + e["azure"]["operation_name"], + "Microsoft.Web/sites/functions/run/action" + ); +} + +// ── Batch limits ────────────────────────────────────────────────────────────── + +/// Exactly MAX_BATCH_ENTRIES entries produce a single batch. +#[tokio::test] +async fn test_max_entries_fits_in_one_batch() { + const MAX: usize = datadog_logs_agent::constants::MAX_BATCH_ENTRIES; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let entries: Vec = (0..MAX).map(|i| entry(&format!("log {i}"))).collect(); + handle.insert_batch(entries).expect("insert"); + + let batches = handle.get_batches().await.expect("get_batches"); + assert_eq!( + batches.len(), + 1, + "exactly MAX_BATCH_ENTRIES fits in one batch" + ); + + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("valid JSON"); + assert_eq!(arr.as_array().unwrap().len(), MAX); +} + +/// MAX_BATCH_ENTRIES + 1 entries split into two batches; two POSTs are sent. +#[tokio::test] +async fn test_overflow_produces_two_batches_and_two_posts() { + const MAX: usize = datadog_logs_agent::constants::MAX_BATCH_ENTRIES; + + let mut server = Server::new_async().await; + let mock = server + .mock("POST", "/logs") + .match_header("DD-API-KEY", "test-api-key") + .with_status(200) + .expect(2) // exactly 2 requests expected + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let entries: Vec = (0..=MAX).map(|i| entry(&format!("log {i}"))).collect(); + handle.insert_batch(entries).expect("insert"); + + let result = LogFlusher::new(opw_config(&server.url()), build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty()); + mock.assert_async().await; +} + +// ── Oversized entries ───────────────────────────────────────────────────────── + +/// Entries exceeding MAX_LOG_BYTES are silently dropped; valid entries still flush. +#[tokio::test] +async fn test_oversized_entry_dropped_valid_entries_still_flush() { + let mut server = Server::new_async().await; + let mock = server + .mock("POST", "/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let oversized = IntakeEntry::from_message( + "x".repeat(datadog_logs_agent::constants::MAX_LOG_BYTES + 1), + 0, + ); + let normal = entry("this one is fine"); + + handle + .insert_batch(vec![oversized, normal]) + .expect("insert"); + + let result = LogFlusher::new(opw_config(&server.url()), build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty(), "flush should succeed for valid entries"); + mock.assert_async().await; +} + +/// All entries oversized means nothing to flush — no HTTP request. +#[tokio::test] +async fn test_all_oversized_entries_produces_no_request() { + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let oversized = IntakeEntry::from_message( + "x".repeat(datadog_logs_agent::constants::MAX_LOG_BYTES + 1), + 0, + ); + handle.insert_batch(vec![oversized]).expect("insert"); + + let batches = handle.get_batches().await.expect("get_batches"); + assert!( + batches.is_empty(), + "oversized-only aggregator should produce no batches" + ); +} + +// ── Concurrent producers ────────────────────────────────────────────────────── + +/// Two cloned handles can insert concurrently; all entries appear in the flush. +#[tokio::test] +async fn test_concurrent_producers_all_entries_flushed() { + let mut server = Server::new_async().await; + let mock = server + .mock("POST", "/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let h1 = handle.clone(); + let h2 = handle.clone(); + + let (r1, r2) = tokio::join!( + tokio::spawn(async move { + h1.insert_batch(vec![entry("from-producer-1")]) + .expect("h1 insert") + }), + tokio::spawn(async move { + h2.insert_batch(vec![entry("from-producer-2")]) + .expect("h2 insert") + }), + ); + r1.expect("task 1"); + r2.expect("task 2"); + + let result = LogFlusher::new(opw_config(&server.url()), build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty()); + mock.assert_async().await; +} + +// ── OPW mode ────────────────────────────────────────────────────────────────── + +/// OPW mode sends to the custom URL and omits the DD-PROTOCOL header. +#[tokio::test] +async fn test_opw_mode_uses_custom_url_and_omits_dd_protocol() { + let mut server = Server::new_async().await; + let opw_path = "/opw-endpoint"; + let mock = server + .mock("POST", opw_path) + .match_header("DD-API-KEY", "test-api-key") + .match_header("DD-PROTOCOL", Matcher::Missing) + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + handle.insert_batch(vec![entry("opw log")]).expect("insert"); + + let config = LogFlusherConfig { + api_key: "test-api-key".to_string(), + site: "ignored".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}{}", server.url(), opw_path), + }, + additional_endpoints: Vec::new(), + use_compression: false, + compression_level: 0, + flush_timeout: Duration::from_secs(5), + }; + + let result = LogFlusher::new(config, build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty()); + mock.assert_async().await; +} + +// ── Compression ─────────────────────────────────────────────────────────────── + +/// OPW mode always disables compression regardless of `use_compression` setting. +/// The request must NOT carry `Content-Encoding: zstd` in OPW mode. +/// +/// Note: zstd compression in Datadog mode is verified in `flusher.rs` unit tests +/// via `ship_batch` directly, since Datadog mode constructs an HTTPS URL that +/// cannot be intercepted by a plain HTTP mock server. +#[tokio::test] +async fn test_opw_mode_disables_compression_regardless_of_config() { + let mut server = Server::new_async().await; + let mock = server + .mock("POST", "/logs") + .match_header("Content-Encoding", Matcher::Missing) // must not be compressed + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + handle + .insert_batch(vec![entry("not compressed in OPW")]) + .expect("insert"); + + // use_compression: true — but OPW mode overrides this to false + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "ignored".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/logs", server.url()), + }, + additional_endpoints: Vec::new(), + use_compression: true, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let result = LogFlusher::new(config, build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty()); + mock.assert_async().await; +} + +// ── Retry behaviour ─────────────────────────────────────────────────────────── + +/// A transient 500 is retried; flush succeeds when the subsequent attempt returns 200. +#[tokio::test] +async fn test_retry_on_500_succeeds_on_second_attempt() { + let mut server = Server::new_async().await; + + let _fail = server + .mock("POST", "/logs") + .with_status(500) + .expect(1) + .create_async() + .await; + let _ok = server + .mock("POST", "/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + handle + .insert_batch(vec![entry("retry me")]) + .expect("insert"); + + let result = LogFlusher::new(opw_config(&server.url()), build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty(), "should succeed after retry"); +} + +/// A 403 is a permanent error; flush fails without additional retry attempts. +#[tokio::test] +async fn test_permanent_error_on_403_no_retry() { + let mut server = Server::new_async().await; + let mock = server + .mock("POST", "/logs") + .with_status(403) + .expect(1) // must be called exactly once — no retries + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + handle + .insert_batch(vec![entry("forbidden")]) + .expect("insert"); + + let result = LogFlusher::new(opw_config(&server.url()), build_client(), handle) + .flush(vec![]) + .await; + + // 403 is a permanent error — dropped silently; no builder to retry. + assert!( + result.is_empty(), + "403 is a permanent error; no retry builder returned" + ); + mock.assert_async().await; +} + +/// All three retry attempts fail with 503; flush returns false. +#[tokio::test] +async fn test_exhausted_retries_returns_false() { + let mut server = Server::new_async().await; + let mock = server + .mock("POST", "/logs") + .with_status(503) + .expect(3) // MAX_FLUSH_ATTEMPTS = 3 + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + handle + .insert_batch(vec![entry("keep failing")]) + .expect("insert"); + + let result = LogFlusher::new(opw_config(&server.url()), build_client(), handle) + .flush(vec![]) + .await; + + // Transient 503 exhausts per-invocation retries; builder returned for next flush. + assert!( + !result.is_empty(), + "exhausted retries should return a retry builder" + ); + mock.assert_async().await; +} + +// ── Additional endpoints ────────────────────────────────────────────────────── + +/// When additional endpoints are configured, the same batch is shipped to each. +#[tokio::test] +async fn test_additional_endpoints_receive_same_batch() { + let mut primary = Server::new_async().await; + let mut secondary = Server::new_async().await; + + let primary_mock = primary + .mock("POST", "/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + + let secondary_mock = secondary + .mock("POST", "/extra") + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + handle + .insert_batch(vec![entry("multi-endpoint")]) + .expect("insert"); + + let config = LogFlusherConfig { + api_key: "test-api-key".to_string(), + site: "ignored".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/logs", primary.url()), + }, + additional_endpoints: vec![LogsAdditionalEndpoint { + api_key: "secondary-api-key".to_string(), + url: format!("{}/extra", secondary.url()), + is_reliable: true, + }], + use_compression: false, + compression_level: 0, + flush_timeout: Duration::from_secs(5), + }; + + let result = LogFlusher::new(config, build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty()); + primary_mock.assert_async().await; + secondary_mock.assert_async().await; +} + +/// Additional endpoint failure does not cause flush() to return false +/// (additional endpoints are best-effort). +#[tokio::test] +async fn test_additional_endpoint_failure_does_not_affect_return_value() { + let mut primary = Server::new_async().await; + let mut secondary = Server::new_async().await; + + let _primary_mock = primary + .mock("POST", "/logs") + .with_status(200) + .create_async() + .await; + + let _secondary_mock = secondary + .mock("POST", "/extra") + .with_status(500) // secondary always fails + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + handle.insert_batch(vec![entry("test")]).expect("insert"); + + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "ignored".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/logs", primary.url()), + }, + additional_endpoints: vec![LogsAdditionalEndpoint { + api_key: "secondary-api-key".to_string(), + url: format!("{}/extra", secondary.url()), + is_reliable: true, + }], + use_compression: false, + compression_level: 0, + flush_timeout: Duration::from_secs(5), + }; + + let result = LogFlusher::new(config, build_client(), handle) + .flush(vec![]) + .await; + + assert!( + result.is_empty(), + "primary succeeded — additional endpoint failure must not affect return value" + ); +} + +// ── Network intake (LogServer) ──────────────────────────────────────────────── + +/// Bind :0 to get a free port, drop the listener, then start LogServer on that +/// port. Returns the base URL ("http://127.0.0.1:"). +async fn start_log_server(handle: datadog_logs_agent::AggregatorHandle) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind :0"); + let port = listener.local_addr().expect("local_addr").port(); + drop(listener); + + let server = LogServer::new( + LogServerConfig { + host: "127.0.0.1".into(), + port, + }, + handle, + ); + tokio::spawn(server.serve()); + sleep(Duration::from_millis(50)).await; // allow server to bind + format!("http://127.0.0.1:{port}") +} + +/// Full network-intake pipeline: HTTP POST → LogServer → AggregatorService → +/// LogFlusher → mockito backend. This mirrors what serverless-compat does +/// when DD_LOGS_ENABLED=true. +#[tokio::test] +async fn test_server_to_flusher_full_pipeline() { + let mut backend = Server::new_async().await; + let mock = backend + .mock("POST", "/logs") + .match_header("DD-API-KEY", "test-api-key") + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + tokio::spawn(svc.run()); + + let base_url = start_log_server(handle.clone()).await; + + // External adapter POSTs a log entry over HTTP (as serverless-compat extension would). + let client = reqwest::Client::new(); + let resp = client + .post(format!("{base_url}/v1/input")) + .json(&serde_json::json!([{ + "message": "invocation start", + "timestamp": 1_700_000_000_000_i64, + "ddsource": "lambda", + "service": "my-fn", + "ddtags": "env:prod" + }])) + .send() + .await + .expect("POST to log server"); + + assert_eq!(resp.status(), 200, "log server should accept the entry"); + + // Flush everything accumulated in the aggregator to the mock backend. + let result = LogFlusher::new(opw_config(&backend.url()), build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty(), "flush should return empty on 200"); + mock.assert_async().await; +} + +/// Multiple concurrent HTTP clients can POST entries simultaneously; all +/// entries must arrive in the aggregator before flushing. +#[tokio::test] +async fn test_server_concurrent_clients_all_entries_arrive() { + let mut backend = Server::new_async().await; + let mock = backend + .mock("POST", "/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + tokio::spawn(svc.run()); + + let base_url = start_log_server(handle.clone()).await; + + // Five concurrent producers each POST one entry. + const N: usize = 5; + let mut tasks = Vec::with_capacity(N); + for i in 0..N { + let url = format!("{base_url}/v1/input"); + tasks.push(tokio::spawn(async move { + reqwest::Client::new() + .post(&url) + .json(&serde_json::json!([{ + "message": format!("entry-{i}"), + "timestamp": i as i64 + }])) + .send() + .await + .expect("concurrent POST") + .status() + })); + } + + for task in tasks { + let status = task.await.expect("task"); + assert_eq!(status, 200); + } + + // All N entries must be present in the aggregator. + let batches = handle.get_batches().await.expect("get_batches"); + let total: usize = batches + .iter() + .map(|b| { + let arr: serde_json::Value = serde_json::from_slice(b).unwrap(); + arr.as_array().unwrap().len() + }) + .sum(); + assert_eq!(total, N, "all {N} concurrent entries must be aggregated"); + + // Re-insert the drained entries so we have something to flush. + let (svc2, handle2) = AggregatorService::new(); + tokio::spawn(svc2.run()); + handle2 + .insert_batch(vec![entry("placeholder")]) + .expect("insert"); + let result = LogFlusher::new(opw_config(&backend.url()), build_client(), handle2) + .flush(vec![]) + .await; + assert!(result.is_empty()); + mock.assert_async().await; +} + +/// A malformed POST (invalid JSON) must return 400 and must not prevent the +/// server from processing subsequent valid requests. +#[tokio::test] +async fn test_server_invalid_request_does_not_block_subsequent_valid_requests() { + let (svc, handle) = AggregatorService::new(); + tokio::spawn(svc.run()); + + let base_url = start_log_server(handle.clone()).await; + let client = reqwest::Client::new(); + let url = format!("{base_url}/v1/input"); + + // Bad JSON → 400 + let bad = client + .post(&url) + .header("Content-Type", "application/json") + .body("not-json-at-all") + .send() + .await + .expect("bad POST"); + assert_eq!(bad.status(), 400); + + // Valid entry immediately after → 200 and entry reaches aggregator + let good = client + .post(&url) + .json(&serde_json::json!([{"message": "after-error", "timestamp": 1_i64}])) + .send() + .await + .expect("good POST"); + assert_eq!(good.status(), 200); + + let batches = handle.get_batches().await.expect("get_batches"); + let total: usize = batches + .iter() + .map(|b| { + let arr: serde_json::Value = serde_json::from_slice(b).unwrap(); + arr.as_array().unwrap().len() + }) + .sum(); + assert_eq!(total, 1, "only the valid entry should be in the aggregator"); +} diff --git a/crates/datadog-serverless-compat/Cargo.toml b/crates/datadog-serverless-compat/Cargo.toml index b41de18..b84bb15 100644 --- a/crates/datadog-serverless-compat/Cargo.toml +++ b/crates/datadog-serverless-compat/Cargo.toml @@ -10,6 +10,7 @@ default = [] windows-pipes = ["datadog-trace-agent/windows-pipes", "dogstatsd/windows-pipes"] [dependencies] +datadog-logs-agent = { path = "../datadog-logs-agent" } datadog-trace-agent = { path = "../datadog-trace-agent" } libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } datadog-fips = { path = "../datadog-fips", default-features = false } @@ -27,5 +28,10 @@ tracing-subscriber = { version = "0.3", default-features = false, features = [ ] } zstd = { version = "0.13.3", default-features = false } +[dev-dependencies] +reqwest = { version = "0.12.4", features = ["json"], default-features = false } +serde_json = { version = "1.0.116", default-features = false, features = ["alloc"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] } + [[bin]] name = "datadog-serverless-compat" diff --git a/crates/datadog-serverless-compat/src/main.rs b/crates/datadog-serverless-compat/src/main.rs index d50798f..8c20c41 100644 --- a/crates/datadog-serverless-compat/src/main.rs +++ b/crates/datadog-serverless-compat/src/main.rs @@ -12,7 +12,7 @@ use tokio::{ sync::Mutex as TokioMutex, time::{Duration, interval}, }; -use tracing::{debug, error, info}; +use tracing::{debug, error, info, warn}; use tracing_subscriber::EnvFilter; use zstd::zstd_safe::CompressionLevel; @@ -26,6 +26,10 @@ use datadog_trace_agent::{ use libdd_trace_utils::{config_utils::read_cloud_env, trace_utils::EnvironmentType}; use datadog_fips::reqwest_adapter::create_reqwest_client_builder; +use datadog_logs_agent::{ + AggregatorHandle as LogAggregatorHandle, AggregatorService as LogAggregatorService, + Destination as LogDestination, LogFlusher, LogFlusherConfig, LogServer, LogServerConfig, +}; use dogstatsd::{ aggregator::{AggregatorHandle, AggregatorService}, api_key::ApiKeyFactory, @@ -42,6 +46,7 @@ use tokio_util::sync::CancellationToken; const DOGSTATSD_FLUSH_INTERVAL: u64 = 10; const DOGSTATSD_TIMEOUT_DURATION: Duration = Duration::from_secs(5); const DEFAULT_DOGSTATSD_PORT: u16 = 8125; +const DEFAULT_LOG_INTAKE_PORT: u16 = 10517; const AGENT_HOST: &str = "0.0.0.0"; #[tokio::main] @@ -107,6 +112,13 @@ pub async fn main() { let https_proxy = env::var("DD_PROXY_HTTPS") .or_else(|_| env::var("HTTPS_PROXY")) .ok(); + let dd_logs_enabled = env::var("DD_LOGS_ENABLED") + .map(|val| val.to_lowercase() == "true") + .unwrap_or(false); + let dd_logs_port: u16 = env::var("DD_LOGS_PORT") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_LOG_INTAKE_PORT); debug!("Starting serverless trace mini agent"); let env_filter = format!("h2=off,hyper=off,rustls=off,{}", log_level); @@ -174,9 +186,9 @@ pub async fn main() { debug!("Starting dogstatsd"); let (_, metrics_flusher, aggregator_handle) = start_dogstatsd( dd_dogstatsd_port, - dd_api_key, + dd_api_key.clone(), dd_site, - https_proxy, + https_proxy.clone(), dogstatsd_tags, dd_statsd_metric_namespace, #[cfg(all(windows, feature = "windows-pipes"))] @@ -194,9 +206,31 @@ pub async fn main() { (None, None) }; + let (log_flusher, _log_aggregator_handle): (Option, Option) = + if dd_logs_enabled { + debug!("Starting log agent"); + match start_log_agent(dd_api_key, https_proxy, dd_logs_port) { + Some((flusher, handle)) => { + info!("log agent started"); + (Some(flusher), Some(handle)) + } + None => { + warn!("log agent failed to start, log flushing disabled"); + (None, None) + } + } + } else { + info!("log agent disabled"); + (None, None) + }; + let mut flush_interval = interval(Duration::from_secs(DOGSTATSD_FLUSH_INTERVAL)); flush_interval.tick().await; // discard first tick, which is instantaneous + // Builders for log batches that failed transiently in the previous flush + // cycle. They are redriven on the next cycle before new batches are sent. + let mut pending_log_retries: Vec = Vec::new(); + loop { flush_interval.tick().await; @@ -204,6 +238,22 @@ pub async fn main() { debug!("Flushing dogstatsd metrics"); metrics_flusher.flush().await; } + + if let Some(log_flusher) = log_flusher.as_ref() { + debug!("Flushing log agent"); + let retry_in = std::mem::take(&mut pending_log_retries); + let failed = log_flusher.flush(retry_in).await; + if !failed.is_empty() { + // TODO: surface flush failures into health/metrics telemetry so + // operators have a durable signal beyond log lines when logs are + // being dropped (e.g. increment a statsd counter or set a gauge). + warn!( + "log agent flush failed for {} batch(es); will retry next cycle", + failed.len() + ); + pending_log_retries = failed; + } + } } } @@ -312,3 +362,187 @@ fn build_metrics_client( } Ok(builder.build()?) } + +fn start_log_agent( + dd_api_key: Option, + https_proxy: Option, + logs_port: u16, +) -> Option<(LogFlusher, LogAggregatorHandle)> { + let Some(api_key) = dd_api_key else { + error!("DD_API_KEY not set, log agent disabled"); + return None; + }; + + let (service, handle): (LogAggregatorService, LogAggregatorHandle) = + LogAggregatorService::new(); + tokio::spawn(service.run()); + + let client = create_reqwest_client_builder() + .map_err(|e| error!("failed to create FIPS HTTP client for log agent: {e}")) + .ok() + .and_then(|b| { + let mut builder = b.timeout(DOGSTATSD_TIMEOUT_DURATION); + if let Some(ref proxy) = https_proxy { + match reqwest::Proxy::https(proxy.as_str()) { + Ok(p) => builder = builder.proxy(p), + Err(e) => error!("invalid HTTPS proxy for log agent: {e}"), + } + } + match builder.build() { + Ok(c) => Some(c), + Err(e) => { + error!("failed to build HTTP client for log agent: {e}"); + None + } + } + }); + + let client = client?; // error already logged above + + let config = LogFlusherConfig { + api_key, + ..LogFlusherConfig::from_env() + }; + + // Fail fast: OPW mode with an empty URL will always produce a network error at flush time. + if let LogDestination::ObservabilityPipelinesWorker { url } = &config.mode + && url.is_empty() + { + error!( + "OPW mode enabled but DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL is empty — log agent disabled" + ); + return None; + } + + // Start the HTTP intake server so external adapters can POST log entries. + let server = LogServer::new( + LogServerConfig { + host: AGENT_HOST.to_string(), + port: logs_port, + }, + handle.clone(), + ); + // TODO(SVLS-bind-fail-fast): `LogServer::serve` binds the port inside the + // spawned task, so any bind failure (e.g. port already in use) is only + // logged as an error and silently swallowed — this function still returns + // `Some(...)` and the caller logs "log agent started" even though the + // server never came up. + // Fix: split `LogServer` into a `bind() -> Result` + // step and a `BoundLogServer::serve()` accept-loop step (both in server.rs). + // Make this fn `async`, call `server.bind().await`, return `None` on error, + // and only spawn `bound.serve()` after a successful bind. Add tests: + // `test_bind_returns_err_when_port_already_in_use` (server.rs) and + // `test_start_log_agent_returns_none_when_port_already_in_use` (main.rs). + tokio::spawn(server.serve()); + info!("log server listening on {AGENT_HOST}:{logs_port}"); + + let flusher = LogFlusher::new(config, client, handle.clone()); + Some((flusher, handle)) +} + +#[cfg(test)] +mod log_agent_integration_tests { + use datadog_logs_agent::{AggregatorService, IntakeEntry, LogServer, LogServerConfig}; + + #[tokio::test] + async fn test_log_agent_full_pipeline_compiles_and_runs() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + handle + .insert_batch(vec![IntakeEntry { + message: "azure function invoked".to_string(), + timestamp: 1_700_000_000_000, + hostname: Some("my-azure-fn".to_string()), + service: Some("payments".to_string()), + ddsource: Some("azure-functions".to_string()), + ddtags: Some("env:prod".to_string()), + status: Some("info".to_string()), + attributes: serde_json::Map::new(), + }]) + .expect("insert_batch"); + + let batches = handle.get_batches().await.expect("get_batches"); + assert_eq!(batches.len(), 1); + + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("json"); + assert_eq!(arr[0]["ddsource"], "azure-functions"); + assert_eq!(arr[0]["service"], "payments"); + + handle.shutdown().expect("shutdown"); + } + + /// start_log_agent must return None when OPW mode is enabled but the URL is empty. + #[tokio::test] + async fn test_opw_empty_url_is_detected() { + use super::start_log_agent; + // Enable OPW mode with a deliberately empty URL — the production guard + // inside start_log_agent must catch this and return None. + // SAFETY: test-only, single-threaded setup before any spawned tasks. + unsafe { + std::env::set_var("DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED", "true"); + std::env::set_var("DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL", ""); + } + let result = start_log_agent(Some("test-key".to_string()), None, 0); + // SAFETY: test-only cleanup. + unsafe { + std::env::remove_var("DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED"); + std::env::remove_var("DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL"); + } + assert!( + result.is_none(), + "start_log_agent must return None when OPW URL is empty" + ); + } + + /// Full network intake path: entries posted over HTTP to LogServer must + /// reach the AggregatorService and be retrievable via get_batches. + /// This mirrors what serverless-compat does when DD_LOGS_ENABLED=true. + #[tokio::test] + #[allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)] + async fn test_log_server_network_intake_end_to_end() { + use tokio::time::{Duration, sleep}; + + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + // Bind :0 to discover a free port, then hand it to LogServer + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + + let server = LogServer::new( + LogServerConfig { + host: "127.0.0.1".into(), + port, + }, + handle.clone(), + ); + tokio::spawn(server.serve()); + sleep(Duration::from_millis(50)).await; + + let client = reqwest::Client::new(); + let resp = client + .post(format!("http://127.0.0.1:{port}/v1/input")) + .json(&serde_json::json!([{ + "message": "lambda function invoked", + "timestamp": 1_700_000_000_000_i64, + "ddsource": "lambda", + "service": "my-fn" + }])) + .send() + .await + .expect("POST to log server failed"); + + assert_eq!(resp.status(), 200, "server must accept the payload"); + + let batches = handle.get_batches().await.expect("get_batches"); + assert_eq!(batches.len(), 1, "one batch expected"); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("json"); + assert_eq!(arr[0]["message"], "lambda function invoked"); + assert_eq!(arr[0]["ddsource"], "lambda"); + assert_eq!(arr[0]["service"], "my-fn"); + + handle.shutdown().expect("shutdown"); + } +} diff --git a/crates/datadog-trace-agent/src/aggregator.rs b/crates/datadog-trace-agent/src/aggregator.rs index 4f36424..b7467ee 100644 --- a/crates/datadog-trace-agent/src/aggregator.rs +++ b/crates/datadog-trace-agent/src/aggregator.rs @@ -104,7 +104,7 @@ mod tests { aggregator.add(payload); assert_eq!(aggregator.queue.len(), 1); - assert_eq!(aggregator.queue[0].is_empty(), false); + assert!(!aggregator.queue[0].is_empty()); assert_eq!(aggregator.queue[0].len(), 1); } diff --git a/crates/datadog-trace-agent/tests/integration_test.rs b/crates/datadog-trace-agent/tests/integration_test.rs index bf28d4f..1491954 100644 --- a/crates/datadog-trace-agent/tests/integration_test.rs +++ b/crates/datadog-trace-agent/tests/integration_test.rs @@ -295,11 +295,11 @@ async fn test_mini_agent_tcp_with_real_flushers() { let mut server_ready = false; for _ in 0..20 { tokio::time::sleep(Duration::from_millis(50)).await; - if let Ok(response) = send_tcp_request(test_port, "/info", "GET", None).await { - if response.status().is_success() { - server_ready = true; - break; - } + if let Ok(response) = send_tcp_request(test_port, "/info", "GET", None).await + && response.status().is_success() + { + server_ready = true; + break; } } assert!( diff --git a/crates/dogstatsd/src/flusher.rs b/crates/dogstatsd/src/flusher.rs index d26579f..70032c6 100644 --- a/crates/dogstatsd/src/flusher.rs +++ b/crates/dogstatsd/src/flusher.rs @@ -231,6 +231,7 @@ async fn should_try_next_batch(resp: Result) -> (bool, } #[cfg(test)] +#[allow(clippy::disallowed_methods)] mod tests { use super::*; use crate::aggregator::AggregatorService; diff --git a/crates/dogstatsd/src/origin.rs b/crates/dogstatsd/src/origin.rs index d0c0952..fc025b9 100644 --- a/crates/dogstatsd/src/origin.rs +++ b/crates/dogstatsd/src/origin.rs @@ -162,16 +162,10 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); + assert_eq!(origin.origin_category, OriginCategory::LambdaMetrics as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, - OriginCategory::LambdaMetrics as u32 - ); - assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessRuntime as u32 ); } @@ -187,16 +181,10 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); + assert_eq!(origin.origin_category, OriginCategory::LambdaMetrics as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, - OriginCategory::LambdaMetrics as u32 - ); - assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessEnhanced as u32 ); } @@ -212,16 +200,10 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); + assert_eq!(origin.origin_category, OriginCategory::LambdaMetrics as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, - OriginCategory::LambdaMetrics as u32 - ); - assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessCustom as u32 ); } @@ -237,16 +219,13 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, + origin.origin_category, OriginCategory::CloudRunMetrics as u32 ); assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessEnhanced as u32 ); } @@ -262,16 +241,13 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, + origin.origin_category, OriginCategory::CloudRunMetrics as u32 ); assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessCustom as u32 ); } @@ -287,16 +263,13 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, + origin.origin_category, OriginCategory::AppServicesMetrics as u32 ); assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessCustom as u32 ); } @@ -312,16 +285,13 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, + origin.origin_category, OriginCategory::ContainerAppMetrics as u32 ); assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessCustom as u32 ); } @@ -337,16 +307,13 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, + origin.origin_category, OriginCategory::AzureFunctionsMetrics as u32 ); assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessCustom as u32 ); } @@ -362,16 +329,10 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); + assert_eq!(origin.origin_category, OriginCategory::LambdaMetrics as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, - OriginCategory::LambdaMetrics as u32 - ); - assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessRuntime as u32 ); } diff --git a/crates/dogstatsd/tests/integration_test.rs b/crates/dogstatsd/tests/integration_test.rs index 3155ef8..f390fa4 100644 --- a/crates/dogstatsd/tests/integration_test.rs +++ b/crates/dogstatsd/tests/integration_test.rs @@ -1,5 +1,6 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +#![allow(clippy::disallowed_methods)] use dogstatsd::metric::SortedTags; use dogstatsd::{ diff --git a/scripts/test-log-intake.sh b/scripts/test-log-intake.sh new file mode 100755 index 0000000..1579aa2 --- /dev/null +++ b/scripts/test-log-intake.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +set -x +# Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +# SPDX-License-Identifier: Apache-2.0 +# +# test-log-intake.sh — Run the log agent example against a local capture server. +# +# The script starts a tiny Python HTTP server that prints every incoming request +# body to stdout so you can inspect the JSON payloads the log agent sends. +# +# USAGE +# # Local capture (default) — no real Datadog traffic: +# ./scripts/test-log-intake.sh +# +# # Send N entries instead of the default 5: +# LOG_ENTRY_COUNT=50 ./scripts/test-log-intake.sh +# +# # Flush to a real Datadog endpoint instead of the local server: +# DD_API_KEY= ./scripts/test-log-intake.sh --real +# +# REQUIREMENTS +# python3 (macOS system python is fine) +# cargo + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKSPACE_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +PORT="${LOG_CAPTURE_PORT:-9999}" +REAL_MODE=false + +for arg in "$@"; do + [[ "$arg" == "--real" ]] && REAL_MODE=true +done + +# ── Build the example first ────────────────────────────────────────────────── +echo "Building send_logs example..." +cargo build -p datadog-logs-agent --example send_logs --quiet 2>&1 + +# ── Real Datadog mode ───────────────────────────────────────────────────────── +if [[ "$REAL_MODE" == true ]]; then + if [[ -z "${DD_API_KEY:-}" ]]; then + echo "Error: DD_API_KEY must be set for --real mode" >&2 + exit 1 + fi + echo "" + echo "Flushing to real Datadog endpoint..." + LOG_ENTRY_COUNT="${LOG_ENTRY_COUNT:-5}" \ + cargo run -p datadog-logs-agent --example send_logs --quiet 2>&1 + exit $? +fi + +# ── Local capture server mode ───────────────────────────────────────────────── + +# Python HTTP server that prints the request body as formatted JSON +CAPTURE_SERVER_SCRIPT=$(cat <<'PYEOF' +import http.server +import json +import sys + +class Handler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + encoding = self.headers.get("Content-Encoding", "none") + content_type = self.headers.get("Content-Type", "") + + print(f"\n{'─'*60}") + print(f"POST {self.path}") + print(f"DD-API-KEY : {self.headers.get('DD-API-KEY', '(not set)')}") + print(f"DD-PROTOCOL: {self.headers.get('DD-PROTOCOL', '(not set)')}") + print(f"Content-Encoding: {encoding}") + print(f"Content-Type : {content_type}") + + if encoding == "zstd": + try: + import zstd + body = zstd.decompress(body) + print("(decompressed zstd payload)") + except ImportError: + print("(zstd payload — install python-zstd to decompress: pip install zstd)") + + if "json" in content_type or body.startswith(b"[") or body.startswith(b"{"): + try: + parsed = json.loads(body) + print(f"\nPayload ({len(parsed) if isinstance(parsed, list) else 1} entries):") + print(json.dumps(parsed, indent=2)) + except json.JSONDecodeError: + print(f"\nRaw body ({len(body)} bytes): {body[:500]}") + else: + print(f"\nRaw body ({len(body)} bytes)") + + self.send_response(200) + self.end_headers() + sys.stdout.flush() + + def log_message(self, fmt, *args): + pass # suppress default access log noise + +port = int(sys.argv[1]) +print(f"Capture server listening on http://localhost:{port}") +print("Waiting for log flush... (Ctrl-C to stop)\n") +sys.stdout.flush() + +httpd = http.server.HTTPServer(("localhost", port), Handler) +httpd.serve_forever() +PYEOF +) + +# Start capture server in background +python3 -c "$CAPTURE_SERVER_SCRIPT" "$PORT" & +SERVER_PID=$! + +cleanup() { + kill "$SERVER_PID" 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +# Give the server a moment to start +sleep 0.3 + +echo "" +echo "Running send_logs example → http://localhost:${PORT}/logs" +echo "" + +DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED=true \ +DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL="http://localhost:${PORT}/logs" \ +DD_API_KEY="${DD_API_KEY:-local-test-key}" \ +LOG_ENTRY_COUNT="${LOG_ENTRY_COUNT:-5}" \ + cargo run -p datadog-logs-agent --example send_logs --quiet 2>&1 + +echo "" +echo "Done. Press Ctrl-C to stop the capture server." +wait "$SERVER_PID" From 96ab942a20979aeb8640214f027cb94e5da7b6fe Mon Sep 17 00:00:00 2001 From: Tianning Li Date: Mon, 6 Apr 2026 16:51:09 -0400 Subject: [PATCH 06/10] ci: remove temporary npm auth token configuration step (#110) The npm token is now handled via the setup-node action's registry-url, making the explicit `npm config set` step redundant. Co-authored-by: Claude Sonnet 4.6 --- .github/workflows/publish.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4dcc42b..4bcb69a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -123,9 +123,6 @@ jobs: with: node-version: "22.x" registry-url: 'https://registry.npmjs.org' - - run: npm config set //registry.npmjs.org/:_authToken=$NPM_PUBLISH_TOKEN - env: - NPM_PUBLISH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} - name: Publish npm packages run: | npm publish ./npm/datadog-serverless-compat-linux-x64 --provenance --access public From 1eb4556919f63656af8a45520fc4f3288bf76bbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 9 Apr 2026 16:18:52 -0400 Subject: [PATCH 07/10] feat(agent-config): allow extensible configuration via ConfigExtension trait (#111) * feat(agent-config): allow extensible configuration via ConfigExtension trait Introduces a generic `Config` type that lets consumers define additional configuration fields without modifying or copy-pasting the core crate. Includes a unified `Source` type for dual extraction from both env vars and YAML, a `merge_fields!` macro to reduce merge boilerplate, and moves Lambda-specific fields out of the core Config struct. Also restructures the crate to use a conventional `src/` layout and adds a README documenting the extension API. * refactor(agent-config): organize crate into sources/ and deserializers/ modules Move config source implementations (env, yaml) into `src/sources/` and type definitions with custom deserialization into `src/deserializers/`. Re-exports at the crate root preserve all existing import paths. * refactor(agent-config): move inline deserializer helpers to deserializers/helpers.rs Extracts all generic deserializer functions (deserialize_optional_string, deserialize_with_default, duration parsers, key-value parsers, etc.) from lib.rs into src/deserializers/helpers.rs. Re-exported at the crate root so all existing import paths continue to work. * refactor(agent-config): reorder lib.rs so Config struct is visible first Reorganize lib.rs so an engineer opening the file immediately sees the Config struct and its fields, followed by the loading entry points, then the extension trait, builder, and macros. Sections are separated with headers for quick scanning. * fix(agent-config): make NoExtensionSource deserialize from map data Change NoExtensionSource from a unit struct to an empty struct so serde accepts map-shaped data from figment (env vars / YAML) instead of expecting null/unit. Prevents spurious warning logs on every get_config() call when no extension is used. * docs(agent-config): improve ConfigExtension trait and Source docs Address reviewer feedback: document field name collision behavior, clarify Source type requirements and their runtime failure modes, and expand the README with collision and flat-field explanations. --- crates/datadog-agent-config/Cargo.toml | 3 - crates/datadog-agent-config/README.md | 117 ++ .../deserializers}/additional_endpoints.rs | 0 .../deserializers}/apm_replace_rule.rs | 0 .../{ => src/deserializers}/flush_strategy.rs | 0 .../src/deserializers/helpers.rs | 372 ++++++ .../{ => src/deserializers}/log_level.rs | 0 .../logs_additional_endpoints.rs | 0 .../src/deserializers/mod.rs | 8 + .../deserializers}/processing_rule.rs | 0 .../deserializers}/service_mapping.rs | 0 .../{mod.rs => src/lib.rs} | 1123 +++++++---------- .../{ => src/sources}/env.rs | 405 +----- .../datadog-agent-config/src/sources/mod.rs | 2 + .../{ => src/sources}/yaml.rs | 158 +-- 15 files changed, 1024 insertions(+), 1164 deletions(-) create mode 100644 crates/datadog-agent-config/README.md rename crates/datadog-agent-config/{ => src/deserializers}/additional_endpoints.rs (100%) rename crates/datadog-agent-config/{ => src/deserializers}/apm_replace_rule.rs (100%) rename crates/datadog-agent-config/{ => src/deserializers}/flush_strategy.rs (100%) create mode 100644 crates/datadog-agent-config/src/deserializers/helpers.rs rename crates/datadog-agent-config/{ => src/deserializers}/log_level.rs (100%) rename crates/datadog-agent-config/{ => src/deserializers}/logs_additional_endpoints.rs (100%) create mode 100644 crates/datadog-agent-config/src/deserializers/mod.rs rename crates/datadog-agent-config/{ => src/deserializers}/processing_rule.rs (100%) rename crates/datadog-agent-config/{ => src/deserializers}/service_mapping.rs (100%) rename crates/datadog-agent-config/{mod.rs => src/lib.rs} (73%) rename crates/datadog-agent-config/{ => src/sources}/env.rs (75%) create mode 100644 crates/datadog-agent-config/src/sources/mod.rs rename crates/datadog-agent-config/{ => src/sources}/yaml.rs (85%) diff --git a/crates/datadog-agent-config/Cargo.toml b/crates/datadog-agent-config/Cargo.toml index 222d726..b9477ac 100644 --- a/crates/datadog-agent-config/Cargo.toml +++ b/crates/datadog-agent-config/Cargo.toml @@ -4,9 +4,6 @@ version = "0.1.0" edition.workspace = true license.workspace = true -[lib] -path = "mod.rs" - [dependencies] figment = { version = "0.10", default-features = false, features = ["yaml", "env"] } libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } diff --git a/crates/datadog-agent-config/README.md b/crates/datadog-agent-config/README.md new file mode 100644 index 0000000..3dd478e --- /dev/null +++ b/crates/datadog-agent-config/README.md @@ -0,0 +1,117 @@ +# datadog-agent-config + +Shared configuration crate for Datadog serverless agents. Provides a typed `Config` struct with built-in loading from environment variables (`DD_*`) and YAML files (`datadog.yaml`), with environment variables taking precedence. + +## Core features + +- **Typed config struct** with fields for site, API key, proxy, logs, APM, metrics, DogStatsD, OTLP, and trace propagation +- **Two built-in sources**: `EnvConfigSource` (reads `DD_*` / `DATADOG_*` env vars) and `YamlConfigSource` (reads `datadog.yaml`) +- **Graceful deserialization**: every field uses forgiving deserializers that fall back to defaults on bad input, so one misconfigured value never crashes the whole config +- **Extensible via `ConfigExtension`**: consumers can define additional configuration fields without modifying this crate + +## Quick start + +```rust +use std::path::Path; +use datadog_agent_config::get_config; + +let config = get_config(Path::new("/var/task")); +println!("site: {}", config.site); +println!("api_key: {}", config.api_key); +``` + +## Extensible configuration + +Consumers that need additional fields (e.g., Lambda-specific settings) implement the `ConfigExtension` trait instead of forking or copy-pasting the crate. + +### 1. Define the extension and its source + +```rust +use datadog_agent_config::{ + ConfigExtension, merge_fields, + deserialize_optional_string, deserialize_optional_bool_from_anything, +}; +use serde::Deserialize; + +#[derive(Debug, PartialEq, Clone)] +pub struct MyExtension { + pub custom_flag: bool, + pub custom_name: String, +} + +impl Default for MyExtension { + fn default() -> Self { + Self { custom_flag: false, custom_name: String::new() } + } +} + +/// Source struct for deserialization. +/// +/// REQUIRED: `#[serde(default)]` on the struct + graceful deserializers on each +/// field. Without these, a missing or malformed value fails the entire extension +/// extraction — fields silently fall back to defaults with a warning log. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(default)] +pub struct MySource { + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub custom_flag: Option, + #[serde(deserialize_with = "deserialize_optional_string")] + pub custom_name: Option, +} + +impl ConfigExtension for MyExtension { + type Source = MySource; + + fn merge_from(&mut self, source: &MySource) { + merge_fields!(self, source, + string: [custom_name], + value: [custom_flag], + ); + } +} +``` + +### 2. Load config with the extension + +```rust +use std::path::Path; +use datadog_agent_config::{Config, get_config_with_extension}; + +type MyConfig = Config; + +let config: MyConfig = get_config_with_extension(Path::new("/var/task")); + +// Core fields +println!("site: {}", config.site); + +// Extension fields +println!("custom_flag: {}", config.ext.custom_flag); +println!("custom_name: {}", config.ext.custom_name); +``` + +Extension fields are populated from both `DD_*` environment variables and `datadog.yaml` using dual extraction: the core fields and extension fields are extracted independently from the same figment instance, so they don't interfere with each other. + +### Flat fields only + +The single `Source` type is used for both env var and YAML extraction. This works because Figment uses a single key-value namespace per provider, so flat fields map naturally to both `DD_*` env vars and top-level YAML keys. If you need nested YAML structures (e.g., `lambda: { enhanced_metrics: true }`) that differ from the flat env var layout, you'd need separate source structs — implement `merge_from` with a nested source struct and handle the mapping manually. + +### Field name collisions + +Extension fields are extracted independently from the same figment as core fields. If an extension defines a field with the same name as a core field (e.g., `api_key`), both get their own copy — they don't interfere, but the extension copy does **not** override the core value. Avoid shadowing core field names to prevent confusion. + +### merge_fields! macro + +The `merge_fields!` macro reduces boilerplate in `merge_from` by batching fields by merge strategy: + +- `string`: merges `Option` into `String` (sets value if `Some`) +- `value`: merges `Option` into `T` (sets value if `Some`) +- `option`: merges `Option` into `Option` (overwrites if `Some`) + +Custom merge logic (e.g., OR-ing two boolean fields together) goes after the macro call in the same method. + +## Config loading precedence + +1. `Config::default()` (hardcoded defaults) +2. `datadog.yaml` values (lower priority) +3. `DD_*` environment variables (highest priority) +4. Post-processing defaults (site, proxy, logs/APM URL construction) diff --git a/crates/datadog-agent-config/additional_endpoints.rs b/crates/datadog-agent-config/src/deserializers/additional_endpoints.rs similarity index 100% rename from crates/datadog-agent-config/additional_endpoints.rs rename to crates/datadog-agent-config/src/deserializers/additional_endpoints.rs diff --git a/crates/datadog-agent-config/apm_replace_rule.rs b/crates/datadog-agent-config/src/deserializers/apm_replace_rule.rs similarity index 100% rename from crates/datadog-agent-config/apm_replace_rule.rs rename to crates/datadog-agent-config/src/deserializers/apm_replace_rule.rs diff --git a/crates/datadog-agent-config/flush_strategy.rs b/crates/datadog-agent-config/src/deserializers/flush_strategy.rs similarity index 100% rename from crates/datadog-agent-config/flush_strategy.rs rename to crates/datadog-agent-config/src/deserializers/flush_strategy.rs diff --git a/crates/datadog-agent-config/src/deserializers/helpers.rs b/crates/datadog-agent-config/src/deserializers/helpers.rs new file mode 100644 index 0000000..058c0ce --- /dev/null +++ b/crates/datadog-agent-config/src/deserializers/helpers.rs @@ -0,0 +1,372 @@ +use serde::{Deserialize, Deserializer}; +use serde_aux::prelude::deserialize_bool_from_anything; +use serde_json::Value; + +use std::collections::HashMap; +use std::fmt; +use std::time::Duration; +use tracing::warn; + +use crate::TracePropagationStyle; + +pub fn deserialize_optional_string<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + match Value::deserialize(deserializer)? { + Value::String(s) => Ok(Some(s)), + other => { + warn!( + "Failed to parse value, expected a string, got: {}, ignoring", + other + ); + Ok(None) + } + } +} + +pub fn deserialize_string_or_int<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Value::deserialize(deserializer)?; + match value { + Value::String(s) => { + if s.trim().is_empty() { + Ok(None) + } else { + Ok(Some(s)) + } + } + Value::Number(n) => Ok(Some(n.to_string())), + _ => { + warn!("Failed to parse value, expected a string or an integer, ignoring"); + Ok(None) + } + } +} + +pub fn deserialize_optional_bool_from_anything<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + // First try to deserialize as Option<_> to handle null/missing values + let opt: Option = Option::deserialize(deserializer)?; + + match opt { + None => Ok(None), + Some(value) => match deserialize_bool_from_anything(value) { + Ok(bool_result) => Ok(Some(bool_result)), + Err(e) => { + warn!("Failed to parse bool value: {}, ignoring", e); + Ok(None) + } + }, + } +} + +/// Parse a single "key:value" string into a (key, value) tuple +/// Returns None if the string is invalid (e.g., missing colon, empty key/value) +fn parse_key_value_tag(tag: &str) -> Option<(String, String)> { + let parts: Vec<&str> = tag.splitn(2, ':').collect(); + if parts.len() == 2 && !parts[0].is_empty() && !parts[1].is_empty() { + Some((parts[0].to_string(), parts[1].to_string())) + } else { + warn!( + "Failed to parse tag '{}', expected format 'key:value', ignoring", + tag + ); + None + } +} + +pub fn deserialize_key_value_pairs<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + struct KeyValueVisitor; + + impl serde::de::Visitor<'_> for KeyValueVisitor { + type Value = HashMap; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string in format 'key1:value1,key2:value2' or 'key1:value1'") + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + let mut map = HashMap::new(); + for tag in value.split(&[',', ' ']) { + if tag.is_empty() { + continue; + } + if let Some((key, val)) = parse_key_value_tag(tag) { + map.insert(key, val); + } + } + + Ok(map) + } + + fn visit_u64(self, value: u64) -> Result + where + E: serde::de::Error, + { + warn!( + "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", + value + ); + Ok(HashMap::new()) + } + + fn visit_i64(self, value: i64) -> Result + where + E: serde::de::Error, + { + warn!( + "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", + value + ); + Ok(HashMap::new()) + } + + fn visit_f64(self, value: f64) -> Result + where + E: serde::de::Error, + { + warn!( + "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", + value + ); + Ok(HashMap::new()) + } + + fn visit_bool(self, value: bool) -> Result + where + E: serde::de::Error, + { + warn!( + "Failed to parse tags: expected string in format 'key:value', got boolean {}, ignoring", + value + ); + Ok(HashMap::new()) + } + } + + deserializer.deserialize_any(KeyValueVisitor) +} + +pub fn deserialize_array_from_comma_separated_string<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let s: String = String::deserialize(deserializer)?; + Ok(s.split(',') + .map(|feature| feature.trim().to_string()) + .filter(|feature| !feature.is_empty()) + .collect()) +} + +pub fn deserialize_key_value_pair_array_to_hashmap<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let array: Vec = match Vec::deserialize(deserializer) { + Ok(v) => v, + Err(e) => { + warn!("Failed to deserialize tags array: {e}, ignoring"); + return Ok(HashMap::new()); + } + }; + let mut map = HashMap::new(); + for s in array { + if let Some((key, val)) = parse_key_value_tag(&s) { + map.insert(key, val); + } + } + Ok(map) +} + +/// Deserialize APM filter tags from space-separated "key:value" pairs, also support key-only tags +pub fn deserialize_apm_filter_tags<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + let opt: Option = Option::deserialize(deserializer)?; + + match opt { + None => Ok(None), + Some(s) if s.trim().is_empty() => Ok(None), + Some(s) => { + let tags: Vec = s + .split_whitespace() + .filter_map(|pair| { + let parts: Vec<&str> = pair.splitn(2, ':').collect(); + if parts.len() == 2 { + let key = parts[0].trim(); + let value = parts[1].trim(); + if key.is_empty() { + None + } else if value.is_empty() { + Some(key.to_string()) + } else { + Some(format!("{key}:{value}")) + } + } else if parts.len() == 1 { + let key = parts[0].trim(); + if key.is_empty() { + None + } else { + Some(key.to_string()) + } + } else { + None + } + }) + .collect(); + + if tags.is_empty() { + Ok(None) + } else { + Ok(Some(tags)) + } + } + } +} + +pub fn deserialize_option_lossless<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + match Option::::deserialize(deserializer) { + Ok(value) => Ok(value), + Err(e) => { + warn!("Failed to deserialize optional value: {}, ignoring", e); + Ok(None) + } + } +} + +/// Gracefully deserialize any field, falling back to `T::default()` on error. +/// +/// This ensures that a single field with the wrong type never fails the entire +/// struct extraction. Works for any `T` that implements `Deserialize + Default`: +/// - `Option` defaults to `None` +/// - `Vec` defaults to `[]` +/// - `HashMap` defaults to `{}` +/// - Structs with `#[derive(Default)]` use their default +pub fn deserialize_with_default<'de, D, T>(deserializer: D) -> Result +where + D: Deserializer<'de>, + T: Deserialize<'de> + Default, +{ + match T::deserialize(deserializer) { + Ok(value) => Ok(value), + Err(e) => { + warn!("Failed to deserialize field: {}, using default", e); + Ok(T::default()) + } + } +} + +pub fn deserialize_optional_duration_from_microseconds<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + match Option::::deserialize(deserializer) { + Ok(opt) => Ok(opt.map(Duration::from_micros)), + Err(e) => { + warn!("Failed to deserialize duration (microseconds): {e}, ignoring"); + Ok(None) + } + } +} + +pub fn deserialize_optional_duration_from_seconds<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + // Deserialize into a generic Value first to avoid propagating type errors, + // then try to extract a duration from it. + match Value::deserialize(deserializer) { + Ok(Value::Number(n)) => { + if let Some(u) = n.as_u64() { + Ok(Some(Duration::from_secs(u))) + } else if let Some(i) = n.as_i64() { + if i < 0 { + warn!("Failed to parse duration: negative durations are not allowed, ignoring"); + Ok(None) + } else { + Ok(Some(Duration::from_secs(i as u64))) + } + } else if let Some(f) = n.as_f64() { + if f < 0.0 { + warn!("Failed to parse duration: negative durations are not allowed, ignoring"); + Ok(None) + } else { + Ok(Some(Duration::from_secs_f64(f))) + } + } else { + warn!("Failed to parse duration: unsupported number format, ignoring"); + Ok(None) + } + } + Ok(Value::Null) => Ok(None), + Ok(other) => { + warn!("Failed to parse duration: expected number, got {other}, ignoring"); + Ok(None) + } + Err(e) => { + warn!("Failed to deserialize duration: {e}, ignoring"); + Ok(None) + } + } +} + +// Like deserialize_optional_duration_from_seconds(), but return None if the value is 0 +pub fn deserialize_optional_duration_from_seconds_ignore_zero<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + let duration: Option = deserialize_optional_duration_from_seconds(deserializer)?; + if duration.is_some_and(|d| d.as_secs() == 0) { + return Ok(None); + } + Ok(duration) +} + +pub fn deserialize_trace_propagation_style<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + use std::str::FromStr; + let s: String = match String::deserialize(deserializer) { + Ok(s) => s, + Err(e) => { + warn!("Failed to deserialize trace propagation style: {e}, ignoring"); + return Ok(Vec::new()); + } + }; + + Ok(s.split(',') + .filter_map( + |style| match TracePropagationStyle::from_str(style.trim()) { + Ok(parsed_style) => Some(parsed_style), + Err(e) => { + warn!("Failed to parse trace propagation style: {e}, ignoring"); + None + } + }, + ) + .collect()) +} diff --git a/crates/datadog-agent-config/log_level.rs b/crates/datadog-agent-config/src/deserializers/log_level.rs similarity index 100% rename from crates/datadog-agent-config/log_level.rs rename to crates/datadog-agent-config/src/deserializers/log_level.rs diff --git a/crates/datadog-agent-config/logs_additional_endpoints.rs b/crates/datadog-agent-config/src/deserializers/logs_additional_endpoints.rs similarity index 100% rename from crates/datadog-agent-config/logs_additional_endpoints.rs rename to crates/datadog-agent-config/src/deserializers/logs_additional_endpoints.rs diff --git a/crates/datadog-agent-config/src/deserializers/mod.rs b/crates/datadog-agent-config/src/deserializers/mod.rs new file mode 100644 index 0000000..e88c90b --- /dev/null +++ b/crates/datadog-agent-config/src/deserializers/mod.rs @@ -0,0 +1,8 @@ +pub mod additional_endpoints; +pub mod apm_replace_rule; +pub mod flush_strategy; +pub mod helpers; +pub mod log_level; +pub mod logs_additional_endpoints; +pub mod processing_rule; +pub mod service_mapping; diff --git a/crates/datadog-agent-config/processing_rule.rs b/crates/datadog-agent-config/src/deserializers/processing_rule.rs similarity index 100% rename from crates/datadog-agent-config/processing_rule.rs rename to crates/datadog-agent-config/src/deserializers/processing_rule.rs diff --git a/crates/datadog-agent-config/service_mapping.rs b/crates/datadog-agent-config/src/deserializers/service_mapping.rs similarity index 100% rename from crates/datadog-agent-config/service_mapping.rs rename to crates/datadog-agent-config/src/deserializers/service_mapping.rs diff --git a/crates/datadog-agent-config/mod.rs b/crates/datadog-agent-config/src/lib.rs similarity index 73% rename from crates/datadog-agent-config/mod.rs rename to crates/datadog-agent-config/src/lib.rs index e8a29bb..f671293 100644 --- a/crates/datadog-agent-config/mod.rs +++ b/crates/datadog-agent-config/src/lib.rs @@ -1,244 +1,45 @@ -pub mod additional_endpoints; -pub mod apm_replace_rule; -pub mod env; -pub mod flush_strategy; -pub mod log_level; -pub mod logs_additional_endpoints; -pub mod processing_rule; -pub mod service_mapping; -pub mod yaml; +pub mod deserializers; +pub mod sources; + +// Re-export submodules at the crate root so existing imports like +// `crate::flush_strategy::FlushStrategy` and `crate::env::EnvConfigSource` keep working. +pub use deserializers::{ + additional_endpoints, apm_replace_rule, flush_strategy, log_level, logs_additional_endpoints, + processing_rule, service_mapping, +}; +pub use sources::{env, yaml}; pub use datadog_opentelemetry::configuration::TracePropagationStyle; +// Re-export all helper deserializers so consumers and internal modules can +// use `crate::deserialize_optional_string` etc. without reaching into submodules. +pub use deserializers::helpers::*; use libdd_trace_obfuscation::replacer::ReplaceRule; use libdd_trace_utils::config_utils::{trace_intake_url, trace_intake_url_prefixed}; -use serde::{Deserialize, Deserializer}; -use serde_aux::prelude::deserialize_bool_from_anything; -use serde_json::Value; +use serde::Deserialize; +use std::collections::HashMap; use std::path::Path; -use std::time::Duration; -use std::{collections::HashMap, fmt}; -use tracing::{debug, error, warn}; +use tracing::{debug, error}; use crate::{ apm_replace_rule::deserialize_apm_replace_rules, env::EnvConfigSource, - flush_strategy::FlushStrategy, log_level::LogLevel, logs_additional_endpoints::LogsAdditionalEndpoint, processing_rule::{ProcessingRule, deserialize_processing_rules}, yaml::YamlConfigSource, }; -/// Helper macro to merge Option fields to String fields -/// -/// Providing one field argument will merge the value from the source config field into the config -/// field. -/// -/// Providing two field arguments will merge the value from the source config field into the config -/// field if the value is not empty. -#[macro_export] -macro_rules! merge_string { - ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { - if let Some(value) = &$source.$source_field { - $config.$config_field.clone_from(value); - } - }; - ($config:expr, $source:expr, $field:ident) => { - if let Some(value) = &$source.$field { - $config.$field.clone_from(value); - } - }; -} - -/// Helper macro to merge Option fields where T implements Clone -/// -/// Providing one field argument will merge the value from the source config field into the config -/// field. -/// -/// Providing two field arguments will merge the value from the source config field into the config -/// field if the value is not empty. -#[macro_export] -macro_rules! merge_option { - ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { - if $source.$source_field.is_some() { - $config.$config_field.clone_from(&$source.$source_field); - } - }; - ($config:expr, $source:expr, $field:ident) => { - if $source.$field.is_some() { - $config.$field.clone_from(&$source.$field); - } - }; -} - -/// Helper macro to merge Option fields to T fields when Option is Some -/// -/// Providing one field argument will merge the value from the source config field into the config -/// field. -/// -/// Providing two field arguments will merge the value from the source config field into the config -/// field if the value is not empty. -#[macro_export] -macro_rules! merge_option_to_value { - ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { - if let Some(value) = &$source.$source_field { - $config.$config_field = value.clone(); - } - }; - ($config:expr, $source:expr, $field:ident) => { - if let Some(value) = &$source.$field { - $config.$field = value.clone(); - } - }; -} - -/// Helper macro to merge `Vec` fields when `Vec` is not empty -/// -/// Providing one field argument will merge the value from the source config field into the config -/// field. -/// -/// Providing two field arguments will merge the value from the source config field into the config -/// field if the value is not empty. -#[macro_export] -macro_rules! merge_vec { - ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { - if !$source.$source_field.is_empty() { - $config.$config_field.clone_from(&$source.$source_field); - } - }; - ($config:expr, $source:expr, $field:ident) => { - if !$source.$field.is_empty() { - $config.$field.clone_from(&$source.$field); - } - }; -} - -// nit: these will replace one map with the other, not merge the maps togehter, right? -/// Helper macro to merge `HashMap` fields when `HashMap` is not empty -/// -/// Providing one field argument will merge the value from the source config field into the config -/// field. -/// -/// Providing two field arguments will merge the value from the source config field into the config -/// field if the value is not empty. -#[macro_export] -macro_rules! merge_hashmap { - ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { - if !$source.$source_field.is_empty() { - $config.$config_field.clone_from(&$source.$source_field); - } - }; - ($config:expr, $source:expr, $field:ident) => { - if !$source.$field.is_empty() { - $config.$field.clone_from(&$source.$field); - } - }; -} - -#[derive(Debug, PartialEq)] -#[allow(clippy::module_name_repetitions)] -pub enum ConfigError { - ParseError(String), - UnsupportedField(String), -} - -#[allow(clippy::module_name_repetitions)] -pub trait ConfigSource { - fn load(&self, config: &mut Config) -> Result<(), ConfigError>; -} - -#[derive(Default)] -#[allow(clippy::module_name_repetitions)] -pub struct ConfigBuilder { - sources: Vec>, - config: Config, -} - -#[allow(clippy::module_name_repetitions)] -impl ConfigBuilder { - #[must_use] - pub fn add_source(mut self, source: Box) -> Self { - self.sources.push(source); - self - } - - pub fn build(&mut self) -> Config { - let mut failed_sources = 0; - for source in &self.sources { - match source.load(&mut self.config) { - Ok(()) => (), - Err(e) => { - error!("Failed to load config: {:?}", e); - failed_sources += 1; - } - } - } - - if !self.sources.is_empty() && failed_sources == self.sources.len() { - debug!("All sources failed to load config, using default config."); - } - - if self.config.site.is_empty() { - self.config.site = "datadoghq.com".to_string(); - } - - // If `proxy_https` is not set, set it from `HTTPS_PROXY` environment variable - // if it exists - if let Ok(https_proxy) = std::env::var("HTTPS_PROXY") - && self.config.proxy_https.is_none() - { - self.config.proxy_https = Some(https_proxy); - } - - // If `proxy_https` is set, check if the site is in `NO_PROXY` environment variable - // or in the `proxy_no_proxy` config field. - if self.config.proxy_https.is_some() { - let site_in_no_proxy = std::env::var("NO_PROXY") - .is_ok_and(|no_proxy| no_proxy.contains(&self.config.site)) - || self - .config - .proxy_no_proxy - .iter() - .any(|no_proxy| no_proxy.contains(&self.config.site)); - if site_in_no_proxy { - self.config.proxy_https = None; - } - } - - // If extraction is not set, set it to the same as the propagation style - if self.config.trace_propagation_style_extract.is_empty() { - self.config - .trace_propagation_style_extract - .clone_from(&self.config.trace_propagation_style); - } - - // If Logs URL is not set, set it to the default - if self.config.logs_config_logs_dd_url.trim().is_empty() { - self.config.logs_config_logs_dd_url = build_fqdn_logs(self.config.site.clone()); - } else { - self.config.logs_config_logs_dd_url = - logs_intake_url(self.config.logs_config_logs_dd_url.as_str()); - } - - // If APM URL is not set, set it to the default - if self.config.apm_dd_url.is_empty() { - self.config.apm_dd_url = trace_intake_url(self.config.site.clone().as_str()); - } else { - // If APM URL is set, add the site to the URL - self.config.apm_dd_url = trace_intake_url_prefixed(self.config.apm_dd_url.as_str()); - } - - self.config.clone() - } -} +// --------------------------------------------------------------------------- +// Config — the resolved configuration struct +// --------------------------------------------------------------------------- #[derive(Debug, PartialEq, Clone)] #[allow(clippy::module_name_repetitions)] #[allow(clippy::struct_excessive_bools)] -pub struct Config { +pub struct Config { pub site: String, pub api_key: String, pub log_level: LogLevel, @@ -349,28 +150,12 @@ pub struct Config { // - Logs pub otlp_config_logs_enabled: bool, - // AWS Lambda - pub api_key_secret_arn: String, - pub kms_api_key: String, - pub api_key_ssm_arn: String, - pub serverless_logs_enabled: bool, - pub serverless_flush_strategy: FlushStrategy, - pub enhanced_metrics: bool, - pub lambda_proc_enhanced_metrics: bool, - pub capture_lambda_payload: bool, - pub capture_lambda_payload_max_depth: u32, - pub compute_trace_stats_on_extension: bool, - pub span_dedup_timeout: Option, - pub api_key_secret_reload_interval: Option, - - pub serverless_appsec_enabled: bool, - pub appsec_rules: Option, - pub appsec_waf_timeout: Duration, - pub api_security_enabled: bool, - pub api_security_sample_delay: Duration, + /// Agent-specific extension fields defined by the consumer. + /// Use `NoExtension` (the default) when no extra fields are needed. + pub ext: E, } -impl Default for Config { +impl Default for Config { fn default() -> Self { Self { site: String::default(), @@ -464,33 +249,30 @@ impl Default for Config { otlp_config_traces_probabilistic_sampler_sampling_percentage: None, otlp_config_logs_enabled: false, - // AWS Lambda - api_key_secret_arn: String::default(), - kms_api_key: String::default(), - api_key_ssm_arn: String::default(), - serverless_logs_enabled: true, - serverless_flush_strategy: FlushStrategy::Default, - enhanced_metrics: true, - lambda_proc_enhanced_metrics: true, - capture_lambda_payload: false, - capture_lambda_payload_max_depth: 10, - compute_trace_stats_on_extension: false, - span_dedup_timeout: None, - api_key_secret_reload_interval: None, - - serverless_appsec_enabled: false, - appsec_rules: None, - appsec_waf_timeout: Duration::from_millis(5), - api_security_enabled: true, - api_security_sample_delay: Duration::from_secs(30), + ext: E::default(), } } } +// --------------------------------------------------------------------------- +// Loading — entry points for building a Config +// --------------------------------------------------------------------------- + #[allow(clippy::module_name_repetitions)] #[inline] #[must_use] pub fn get_config(config_directory: &Path) -> Config { + get_config_with_extension(config_directory) +} + +/// Load configuration with a custom extension type. +/// +/// Consumers that need additional fields should call this with their +/// extension type instead of `get_config`. +#[allow(clippy::module_name_repetitions)] +#[inline] +#[must_use] +pub fn get_config_with_extension(config_directory: &Path) -> Config { let path: std::path::PathBuf = config_directory.join("datadog.yaml"); ConfigBuilder::default() .add_source(Box::new(YamlConfigSource { path })) @@ -498,385 +280,353 @@ pub fn get_config(config_directory: &Path) -> Config { .build() } -#[inline] -#[must_use] -fn build_fqdn_logs(site: String) -> String { - format!("https://http-intake.logs.{site}") +// --------------------------------------------------------------------------- +// ConfigExtension — trait for additional configuration fields +// --------------------------------------------------------------------------- + +/// Trait that extension configs must implement to add additional configuration +/// fields beyond what the core provides. +/// +/// Extensions allow consumers to define their own external configuration fields +/// that are deserialized from environment variables and YAML files alongside +/// core fields via dual extraction. +/// +/// # Source type requirements +/// +/// The `Source` type **must** use `#[serde(default)]` on the struct and graceful +/// deserializers (e.g., `deserialize_optional_bool_from_anything`) on each field. +/// Without these, a missing or malformed value will cause the entire extension +/// extraction to fail — the extension silently falls back to `E::default()` with +/// a `tracing::warn!` log. See [`ConfigExtension::Source`] for details. +/// +/// # Flat fields only +/// +/// A single `Source` type is used for both environment variable and YAML +/// extraction. This works when all extension fields are top-level (flat) in +/// the YAML file, which is the common case for extension configs: +/// +/// ```yaml +/// # Works: flat fields map naturally to both DD_* env vars and YAML keys +/// enhanced_metrics: true +/// capture_lambda_payload: false +/// ``` +/// +/// If you need nested YAML structures (e.g., `lambda: { enhanced_metrics: true }`) +/// that differ from the flat env var layout, implement `merge_from` with a +/// nested source struct and handle the mapping manually instead of using +/// `merge_fields!`. +/// +/// # Field name collisions with core config +/// +/// Extension fields are extracted independently from the same figment as core +/// fields. If an extension defines a field with the same name as a core field +/// (e.g., `api_key`), both will deserialize their own copy — they do not +/// interfere with each other, but the extension copy will **not** override the +/// core value. Avoid shadowing core field names to prevent confusion. +pub trait ConfigExtension: Clone + Default + std::fmt::Debug + PartialEq { + /// Intermediate deserialization type for extension fields, used for both + /// environment variable and YAML extraction. + /// + /// # Requirements + /// + /// The struct **must** have: + /// + /// 1. `#[serde(default)]` on the struct — so missing fields get defaults + /// instead of failing the whole extraction. + /// 2. Graceful per-field deserializers (e.g., + /// `#[serde(deserialize_with = "deserialize_optional_bool_from_anything")]`) + /// — so one malformed value doesn't fail the whole extraction. + /// + /// **If either is missing**, `figment::extract::()` will fail at + /// runtime when a field is absent or malformed. The extension falls back to + /// `E::default()` and a `tracing::warn!` is emitted — no panic, but all + /// extension fields silently get their default values. + type Source: Default + serde::de::DeserializeOwned + Clone + std::fmt::Debug; + + /// Merge parsed source fields into self. + fn merge_from(&mut self, source: &Self::Source); } -#[inline] -#[must_use] -fn logs_intake_url(url: &str) -> String { - let url = url.trim(); - if url.is_empty() { - return url.to_string(); - } - if url.starts_with("https://") || url.starts_with("http://") { - return url.to_string(); - } - format!("https://{url}") +/// A no-op extension for consumers that don't need extra fields. +#[derive(Clone, Default, Debug, PartialEq)] +pub struct NoExtension; + +/// A no-op source for deserialization that accepts (and ignores) any input. +/// Uses a regular struct (not unit struct) so serde deserializes it from +/// map-shaped data that figment provides, rather than expecting null/unit. +#[derive(Clone, Default, Debug, Deserialize)] +pub struct NoExtensionSource {} + +impl ConfigExtension for NoExtension { + type Source = NoExtensionSource; + fn merge_from(&mut self, _source: &Self::Source) {} } -pub fn deserialize_optional_string<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - match Value::deserialize(deserializer)? { - Value::String(s) => Ok(Some(s)), - other => { - warn!( - "Failed to parse value, expected a string, got: {}, ignoring", - other - ); - Ok(None) - } - } +// --------------------------------------------------------------------------- +// ConfigBuilder — orchestrates loading from multiple sources +// --------------------------------------------------------------------------- + +#[derive(Debug, PartialEq)] +#[allow(clippy::module_name_repetitions)] +pub enum ConfigError { + ParseError(String), + UnsupportedField(String), } -pub fn deserialize_string_or_int<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let value = Value::deserialize(deserializer)?; - match value { - Value::String(s) => { - if s.trim().is_empty() { - Ok(None) - } else { - Ok(Some(s)) - } - } - Value::Number(n) => Ok(Some(n.to_string())), - _ => { - warn!("Failed to parse value, expected a string or an integer, ignoring"); - Ok(None) - } - } +#[allow(clippy::module_name_repetitions)] +pub trait ConfigSource { + fn load(&self, config: &mut Config) -> Result<(), ConfigError>; } -pub fn deserialize_optional_bool_from_anything<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - // First try to deserialize as Option<_> to handle null/missing values - let opt: Option = Option::deserialize(deserializer)?; - - match opt { - None => Ok(None), - Some(value) => match deserialize_bool_from_anything(value) { - Ok(bool_result) => Ok(Some(bool_result)), - Err(e) => { - warn!("Failed to parse bool value: {}, ignoring", e); - Ok(None) - } - }, - } +#[allow(clippy::module_name_repetitions)] +pub struct ConfigBuilder { + sources: Vec>>, + config: Config, } -/// Parse a single "key:value" string into a (key, value) tuple -/// Returns None if the string is invalid (e.g., missing colon, empty key/value) -fn parse_key_value_tag(tag: &str) -> Option<(String, String)> { - let parts: Vec<&str> = tag.splitn(2, ':').collect(); - if parts.len() == 2 && !parts[0].is_empty() && !parts[1].is_empty() { - Some((parts[0].to_string(), parts[1].to_string())) - } else { - warn!( - "Failed to parse tag '{}', expected format 'key:value', ignoring", - tag - ); - None +impl Default for ConfigBuilder { + fn default() -> Self { + Self { + sources: Vec::new(), + config: Config::default(), + } } } -pub fn deserialize_key_value_pairs<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - struct KeyValueVisitor; - - impl serde::de::Visitor<'_> for KeyValueVisitor { - type Value = HashMap; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a string in format 'key1:value1,key2:value2' or 'key1:value1'") - } +#[allow(clippy::module_name_repetitions)] +impl ConfigBuilder { + #[must_use] + pub fn add_source(mut self, source: Box>) -> Self { + self.sources.push(source); + self + } - fn visit_str(self, value: &str) -> Result - where - E: serde::de::Error, - { - let mut map = HashMap::new(); - for tag in value.split(&[',', ' ']) { - if tag.is_empty() { - continue; - } - if let Some((key, val)) = parse_key_value_tag(tag) { - map.insert(key, val); + pub fn build(&mut self) -> Config { + let mut failed_sources = 0; + for source in &self.sources { + match source.load(&mut self.config) { + Ok(()) => (), + Err(e) => { + error!("Failed to load config: {:?}", e); + failed_sources += 1; } } + } - Ok(map) + if !self.sources.is_empty() && failed_sources == self.sources.len() { + debug!("All sources failed to load config, using default config."); } - fn visit_u64(self, value: u64) -> Result - where - E: serde::de::Error, - { - warn!( - "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", - value - ); - Ok(HashMap::new()) + if self.config.site.is_empty() { + self.config.site = "datadoghq.com".to_string(); } - fn visit_i64(self, value: i64) -> Result - where - E: serde::de::Error, + // If `proxy_https` is not set, set it from `HTTPS_PROXY` environment variable + // if it exists + if let Ok(https_proxy) = std::env::var("HTTPS_PROXY") + && self.config.proxy_https.is_none() { - warn!( - "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", - value - ); - Ok(HashMap::new()) + self.config.proxy_https = Some(https_proxy); } - fn visit_f64(self, value: f64) -> Result - where - E: serde::de::Error, - { - warn!( - "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", - value - ); - Ok(HashMap::new()) + // If `proxy_https` is set, check if the site is in `NO_PROXY` environment variable + // or in the `proxy_no_proxy` config field. + if self.config.proxy_https.is_some() { + let site_in_no_proxy = std::env::var("NO_PROXY") + .is_ok_and(|no_proxy| no_proxy.contains(&self.config.site)) + || self + .config + .proxy_no_proxy + .iter() + .any(|no_proxy| no_proxy.contains(&self.config.site)); + if site_in_no_proxy { + self.config.proxy_https = None; + } + } + + // If extraction is not set, set it to the same as the propagation style + if self.config.trace_propagation_style_extract.is_empty() { + self.config + .trace_propagation_style_extract + .clone_from(&self.config.trace_propagation_style); + } + + // If Logs URL is not set, set it to the default + if self.config.logs_config_logs_dd_url.trim().is_empty() { + self.config.logs_config_logs_dd_url = build_fqdn_logs(self.config.site.clone()); + } else { + self.config.logs_config_logs_dd_url = + logs_intake_url(self.config.logs_config_logs_dd_url.as_str()); } - fn visit_bool(self, value: bool) -> Result - where - E: serde::de::Error, - { - warn!( - "Failed to parse tags: expected string in format 'key:value', got boolean {}, ignoring", - value - ); - Ok(HashMap::new()) + // If APM URL is not set, set it to the default + if self.config.apm_dd_url.is_empty() { + self.config.apm_dd_url = trace_intake_url(self.config.site.clone().as_str()); + } else { + // If APM URL is set, add the site to the URL + self.config.apm_dd_url = trace_intake_url_prefixed(self.config.apm_dd_url.as_str()); } + + self.config.clone() } +} - deserializer.deserialize_any(KeyValueVisitor) +#[inline] +#[must_use] +fn build_fqdn_logs(site: String) -> String { + format!("https://http-intake.logs.{site}") } -pub fn deserialize_array_from_comma_separated_string<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let s: String = String::deserialize(deserializer)?; - Ok(s.split(',') - .map(|feature| feature.trim().to_string()) - .filter(|feature| !feature.is_empty()) - .collect()) +#[inline] +#[must_use] +fn logs_intake_url(url: &str) -> String { + let url = url.trim(); + if url.is_empty() { + return url.to_string(); + } + if url.starts_with("https://") || url.starts_with("http://") { + return url.to_string(); + } + format!("https://{url}") } -pub fn deserialize_key_value_pair_array_to_hashmap<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let array: Vec = match Vec::deserialize(deserializer) { - Ok(v) => v, - Err(e) => { - warn!("Failed to deserialize tags array: {e}, ignoring"); - return Ok(HashMap::new()); +// --------------------------------------------------------------------------- +// Merge macros — used by sources and extension implementations +// --------------------------------------------------------------------------- + +/// Helper macro to merge Option fields to String fields +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. +/// +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_string { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if let Some(value) = &$source.$source_field { + $config.$config_field.clone_from(value); } }; - let mut map = HashMap::new(); - for s in array { - if let Some((key, val)) = parse_key_value_tag(&s) { - map.insert(key, val); + ($config:expr, $source:expr, $field:ident) => { + if let Some(value) = &$source.$field { + $config.$field.clone_from(value); } - } - Ok(map) + }; } -/// Deserialize APM filter tags from space-separated "key:value" pairs, also support key-only tags -pub fn deserialize_apm_filter_tags<'de, D>(deserializer: D) -> Result>, D::Error> -where - D: Deserializer<'de>, -{ - let opt: Option = Option::deserialize(deserializer)?; - - match opt { - None => Ok(None), - Some(s) if s.trim().is_empty() => Ok(None), - Some(s) => { - let tags: Vec = s - .split_whitespace() - .filter_map(|pair| { - let parts: Vec<&str> = pair.splitn(2, ':').collect(); - if parts.len() == 2 { - let key = parts[0].trim(); - let value = parts[1].trim(); - if key.is_empty() { - None - } else if value.is_empty() { - Some(key.to_string()) - } else { - Some(format!("{key}:{value}")) - } - } else if parts.len() == 1 { - let key = parts[0].trim(); - if key.is_empty() { - None - } else { - Some(key.to_string()) - } - } else { - None - } - }) - .collect(); - - if tags.is_empty() { - Ok(None) - } else { - Ok(Some(tags)) - } +/// Helper macro to merge Option fields where T implements Clone +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. +/// +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_option { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if $source.$source_field.is_some() { + $config.$config_field.clone_from(&$source.$source_field); } - } -} - -pub fn deserialize_option_lossless<'de, D, T>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, - T: Deserialize<'de>, -{ - match Option::::deserialize(deserializer) { - Ok(value) => Ok(value), - Err(e) => { - warn!("Failed to deserialize optional value: {}, ignoring", e); - Ok(None) + }; + ($config:expr, $source:expr, $field:ident) => { + if $source.$field.is_some() { + $config.$field.clone_from(&$source.$field); } - } + }; } -/// Gracefully deserialize any field, falling back to `T::default()` on error. +/// Helper macro to merge Option fields to T fields when Option is Some +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. /// -/// This ensures that a single field with the wrong type never fails the entire -/// struct extraction. Works for any `T` that implements `Deserialize + Default`: -/// - `Option` defaults to `None` -/// - `Vec` defaults to `[]` -/// - `HashMap` defaults to `{}` -/// - Structs with `#[derive(Default)]` use their default -pub fn deserialize_with_default<'de, D, T>(deserializer: D) -> Result -where - D: Deserializer<'de>, - T: Deserialize<'de> + Default, -{ - match T::deserialize(deserializer) { - Ok(value) => Ok(value), - Err(e) => { - warn!("Failed to deserialize field: {}, using default", e); - Ok(T::default()) +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_option_to_value { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if let Some(value) = &$source.$source_field { + $config.$config_field = value.clone(); } - } -} - -pub fn deserialize_optional_duration_from_microseconds<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result, D::Error> { - match Option::::deserialize(deserializer) { - Ok(opt) => Ok(opt.map(Duration::from_micros)), - Err(e) => { - warn!("Failed to deserialize duration (microseconds): {e}, ignoring"); - Ok(None) + }; + ($config:expr, $source:expr, $field:ident) => { + if let Some(value) = &$source.$field { + $config.$field = value.clone(); } - } + }; } -pub fn deserialize_optional_duration_from_seconds<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result, D::Error> { - // Deserialize into a generic Value first to avoid propagating type errors, - // then try to extract a duration from it. - match Value::deserialize(deserializer) { - Ok(Value::Number(n)) => { - if let Some(u) = n.as_u64() { - Ok(Some(Duration::from_secs(u))) - } else if let Some(i) = n.as_i64() { - if i < 0 { - warn!("Failed to parse duration: negative durations are not allowed, ignoring"); - Ok(None) - } else { - Ok(Some(Duration::from_secs(i as u64))) - } - } else if let Some(f) = n.as_f64() { - if f < 0.0 { - warn!("Failed to parse duration: negative durations are not allowed, ignoring"); - Ok(None) - } else { - Ok(Some(Duration::from_secs_f64(f))) - } - } else { - warn!("Failed to parse duration: unsupported number format, ignoring"); - Ok(None) - } - } - Ok(Value::Null) => Ok(None), - Ok(other) => { - warn!("Failed to parse duration: expected number, got {other}, ignoring"); - Ok(None) +/// Helper macro to merge `Vec` fields when `Vec` is not empty +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. +/// +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_vec { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if !$source.$source_field.is_empty() { + $config.$config_field.clone_from(&$source.$source_field); } - Err(e) => { - warn!("Failed to deserialize duration: {e}, ignoring"); - Ok(None) + }; + ($config:expr, $source:expr, $field:ident) => { + if !$source.$field.is_empty() { + $config.$field.clone_from(&$source.$field); } - } -} - -// Like deserialize_optional_duration_from_seconds(), but return None if the value is 0 -pub fn deserialize_optional_duration_from_seconds_ignore_zero<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result, D::Error> { - let duration: Option = deserialize_optional_duration_from_seconds(deserializer)?; - if duration.is_some_and(|d| d.as_secs() == 0) { - return Ok(None); - } - Ok(duration) + }; } -pub fn deserialize_trace_propagation_style<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - use std::str::FromStr; - let s: String = match String::deserialize(deserializer) { - Ok(s) => s, - Err(e) => { - warn!("Failed to deserialize trace propagation style: {e}, ignoring"); - return Ok(Vec::new()); +/// Helper macro to merge `HashMap` fields when `HashMap` is not empty +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. +/// +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_hashmap { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if !$source.$source_field.is_empty() { + $config.$config_field.clone_from(&$source.$source_field); + } + }; + ($config:expr, $source:expr, $field:ident) => { + if !$source.$field.is_empty() { + $config.$field.clone_from(&$source.$field); } }; +} - Ok(s.split(',') - .filter_map( - |style| match TracePropagationStyle::from_str(style.trim()) { - Ok(parsed_style) => Some(parsed_style), - Err(e) => { - warn!("Failed to parse trace propagation style: {e}, ignoring"); - None - } - }, - ) - .collect()) +/// Batch-merge extension fields from a source struct. +/// +/// Groups fields by merge strategy so you don't have to write individual +/// `merge_string!` / `merge_option_to_value!` / `merge_option!` calls. +/// +/// ```ignore +/// merge_fields!(self, source, +/// string: [api_key_secret_arn, kms_api_key], +/// value: [enhanced_metrics, capture_lambda_payload], +/// option: [span_dedup_timeout, appsec_rules], +/// ); +/// ``` +#[macro_export] +macro_rules! merge_fields { + // Internal rules dispatched by keyword + (@string $config:expr, $source:expr, [$($field:ident),* $(,)?]) => { + $( $crate::merge_string!($config, $source, $field); )* + }; + (@value $config:expr, $source:expr, [$($field:ident),* $(,)?]) => { + $( $crate::merge_option_to_value!($config, $source, $field); )* + }; + (@option $config:expr, $source:expr, [$($field:ident),* $(,)?]) => { + $( $crate::merge_option!($config, $source, $field); )* + }; + // Public entry point: accepts any combination of groups in any order + ($config:expr, $source:expr, $($kind:ident: [$($field:ident),* $(,)?]),* $(,)?) => { + $( $crate::merge_fields!(@$kind $config, $source, [$($field),*]); )* + }; } #[cfg_attr(coverage_nightly, coverage(off))] // Test modules skew coverage metrics @@ -887,12 +637,9 @@ pub mod tests { use super::*; - use crate::{ - TracePropagationStyle, - flush_strategy::{FlushStrategy, PeriodicStrategy}, - log_level::LogLevel, - processing_rule::ProcessingRule, - }; + use std::time::Duration; + + use crate::{TracePropagationStyle, log_level::LogLevel, processing_rule::ProcessingRule}; #[test] fn test_default_logs_intake_url() { @@ -1159,56 +906,6 @@ pub mod tests { }); } - #[test] - fn test_parse_flush_strategy_end() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "end"); - let config = get_config(Path::new("")); - assert_eq!(config.serverless_flush_strategy, FlushStrategy::End); - Ok(()) - }); - } - - #[test] - fn test_parse_flush_strategy_periodically() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,100000"); - let config = get_config(Path::new("")); - assert_eq!( - config.serverless_flush_strategy, - FlushStrategy::Periodically(PeriodicStrategy { interval: 100_000 }) - ); - Ok(()) - }); - } - - #[test] - fn test_parse_flush_strategy_invalid() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "invalid_strategy"); - let config = get_config(Path::new("")); - assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); - Ok(()) - }); - } - - #[test] - fn test_parse_flush_strategy_invalid_periodic() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env( - "DD_SERVERLESS_FLUSH_STRATEGY", - "periodically,invalid_interval", - ); - let config = get_config(Path::new("")); - assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); - Ok(()) - }); - } - #[test] fn parse_number_or_string_env_vars() { figment::Jail::expect_with(|jail| { @@ -1477,15 +1174,11 @@ pub mod tests { fn test_parse_bool_from_anything() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "true"); - jail.set_env("DD_ENHANCED_METRICS", "1"); jail.set_env("DD_LOGS_CONFIG_USE_COMPRESSION", "TRUE"); - jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "0"); + jail.set_env("DD_SKIP_SSL_VALIDATION", "1"); let config = get_config(Path::new("")); - assert!(config.serverless_logs_enabled); - assert!(config.enhanced_metrics); assert!(config.logs_config_use_compression); - assert!(!config.capture_lambda_payload); + assert!(config.skip_ssl_validation); Ok(()) }); } @@ -1709,4 +1402,144 @@ pub mod tests { serde_json::from_str::(r#"{"tags": []}"#).expect("failed to parse JSON"); assert_eq!(result.tags, HashMap::new()); } + + // -- ConfigExtension tests -- + + /// A test extension with a few fields, mimicking what a consumer like Lambda would define. + #[derive(Clone, Default, Debug, PartialEq)] + struct TestExtension { + custom_flag: bool, + custom_name: String, + } + + #[derive(Clone, Default, Debug, Deserialize)] + #[serde(default)] + struct TestExtSource { + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + custom_flag: Option, + #[serde(deserialize_with = "deserialize_optional_string")] + custom_name: Option, + } + + impl ConfigExtension for TestExtension { + type Source = TestExtSource; + + fn merge_from(&mut self, source: &TestExtSource) { + merge_fields!(self, source, + string: [custom_name], + value: [custom_flag], + ); + } + } + + #[test] + fn test_no_extension_config_works() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_SITE", "datad0g.com"); + let config = get_config(Path::new("")); + assert_eq!(config.site, "datad0g.com"); + assert_eq!(config.ext, NoExtension); + Ok(()) + }); + } + + #[test] + fn test_extension_receives_env_vars() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_SITE", "datad0g.com"); + jail.set_env("DD_CUSTOM_FLAG", "true"); + jail.set_env("DD_CUSTOM_NAME", "my-extension"); + + let config: Config = get_config_with_extension(Path::new("")); + + // Core fields work + assert_eq!(config.site, "datad0g.com"); + // Extension fields are populated + assert!(config.ext.custom_flag); + assert_eq!(config.ext.custom_name, "my-extension"); + Ok(()) + }); + } + + #[test] + fn test_extension_receives_yaml_fields() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r#" +site: "datad0g.com" +custom_flag: true +custom_name: "yaml-ext" +"#, + )?; + + let config: Config = get_config_with_extension(Path::new("")); + + assert_eq!(config.site, "datad0g.com"); + assert!(config.ext.custom_flag); + assert_eq!(config.ext.custom_name, "yaml-ext"); + Ok(()) + }); + } + + #[test] + fn test_extension_env_overrides_yaml() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r#" +custom_name: "yaml-value" +custom_flag: false +"#, + )?; + jail.set_env("DD_CUSTOM_NAME", "env-value"); + jail.set_env("DD_CUSTOM_FLAG", "true"); + + let config: Config = get_config_with_extension(Path::new("")); + + // Env should override YAML (env source loaded after yaml) + assert!(config.ext.custom_flag); + assert_eq!(config.ext.custom_name, "env-value"); + Ok(()) + }); + } + + #[test] + fn test_extension_defaults_when_not_set() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + + let config: Config = get_config_with_extension(Path::new("")); + + // Extension fields should be at their defaults + assert!(!config.ext.custom_flag); + assert_eq!(config.ext.custom_name, ""); + // Core fields should have post-processing defaults + assert_eq!(config.site, "datadoghq.com"); + Ok(()) + }); + } + + #[test] + fn test_extension_does_not_interfere_with_core() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_SITE", "us5.datadoghq.com"); + jail.set_env("DD_API_KEY", "test-key"); + jail.set_env("DD_CUSTOM_FLAG", "true"); + + let config: Config = get_config_with_extension(Path::new("")); + + // Core fields are not affected by extension env vars + assert_eq!(config.site, "us5.datadoghq.com"); + assert_eq!(config.api_key, "test-key"); + // Extension fields work alongside core + assert!(config.ext.custom_flag); + Ok(()) + }); + } } diff --git a/crates/datadog-agent-config/env.rs b/crates/datadog-agent-config/src/sources/env.rs similarity index 75% rename from crates/datadog-agent-config/env.rs rename to crates/datadog-agent-config/src/sources/env.rs index 4dae58b..6fb96a1 100644 --- a/crates/datadog-agent-config/env.rs +++ b/crates/datadog-agent-config/src/sources/env.rs @@ -1,22 +1,18 @@ use figment::{Figment, providers::Env}; use serde::Deserialize; use std::collections::HashMap; -use std::time::Duration; use dogstatsd::util::parse_metric_namespace; use libdd_trace_obfuscation::replacer::ReplaceRule; use crate::{ - Config, ConfigError, ConfigSource, TracePropagationStyle, + Config, ConfigError, ConfigExtension, ConfigSource, TracePropagationStyle, additional_endpoints::deserialize_additional_endpoints, apm_replace_rule::deserialize_apm_replace_rules, deserialize_apm_filter_tags, deserialize_array_from_comma_separated_string, deserialize_key_value_pairs, deserialize_option_lossless, - deserialize_optional_bool_from_anything, deserialize_optional_duration_from_microseconds, - deserialize_optional_duration_from_seconds, - deserialize_optional_duration_from_seconds_ignore_zero, deserialize_optional_string, + deserialize_optional_bool_from_anything, deserialize_optional_string, deserialize_string_or_int, deserialize_trace_propagation_style, deserialize_with_default, - flush_strategy::FlushStrategy, log_level::LogLevel, logs_additional_endpoints::{LogsAdditionalEndpoint, deserialize_logs_additional_endpoints}, merge_hashmap, merge_option, merge_option_to_value, merge_string, merge_vec, @@ -369,119 +365,10 @@ pub struct EnvConfig { /// @env `DD_OTLP_CONFIG_LOGS_ENABLED` #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub otlp_config_logs_enabled: Option, - - // AWS Lambda - /// @env `DD_API_KEY_SECRET_ARN` - /// - /// The AWS ARN of the secret containing the Datadog API key. - #[serde(deserialize_with = "deserialize_optional_string")] - pub api_key_secret_arn: Option, - /// @env `DD_KMS_API_KEY` - /// - /// The AWS KMS API key to use for the Datadog Agent. - #[serde(deserialize_with = "deserialize_optional_string")] - pub kms_api_key: Option, - /// @env `DD_API_KEY_SSM_ARN` - /// - /// The AWS Systems Manager Parameter Store parameter ARN containing the Datadog API key. - #[serde(deserialize_with = "deserialize_optional_string")] - pub api_key_ssm_arn: Option, - /// @env `DD_SERVERLESS_LOGS_ENABLED` - /// - /// Enable logs for AWS Lambda. Default is `true`. - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub serverless_logs_enabled: Option, - /// @env `DD_LOGS_ENABLED` - /// - /// Enable logs for AWS Lambda. Alias for `DD_SERVERLESS_LOGS_ENABLED`. Default is `true`. - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub logs_enabled: Option, - /// @env `DD_SERVERLESS_FLUSH_STRATEGY` - /// - /// The flush strategy to use for AWS Lambda. - #[serde(deserialize_with = "deserialize_with_default")] - pub serverless_flush_strategy: Option, - /// @env `DD_ENHANCED_METRICS` - /// - /// Enable enhanced metrics for AWS Lambda. Default is `true`. - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub enhanced_metrics: Option, - /// @env `DD_LAMBDA_PROC_ENHANCED_METRICS` - /// - /// Enable Lambda process metrics for AWS Lambda. Default is `true`. - /// - /// This is for metrics like: - /// - CPU usage - /// - Network usage - /// - File descriptor count - /// - Thread count - /// - Temp directory usage - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub lambda_proc_enhanced_metrics: Option, - /// @env `DD_CAPTURE_LAMBDA_PAYLOAD` - /// - /// Enable capture of the Lambda request and response payloads. - /// Default is `false`. - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub capture_lambda_payload: Option, - /// @env `DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH` - /// - /// The maximum depth of the Lambda payload to capture. - /// Default is `10`. Requires `capture_lambda_payload` to be `true`. - #[serde(deserialize_with = "deserialize_option_lossless")] - pub capture_lambda_payload_max_depth: Option, - /// @env `DD_COMPUTE_TRACE_STATS_ON_EXTENSION` - /// - /// If true, enable computation of trace stats on the extension side. - /// If false, trace stats will be computed on the backend side. - /// Default is `false`. - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub compute_trace_stats_on_extension: Option, - /// @env `DD_SPAN_DEDUP_TIMEOUT` - /// - /// The timeout for the span deduplication service to check if a span key exists, in seconds. - /// For now, this is a temporary field added to debug the failure of `check_and_add()` in span dedup service. - /// Do not use this field extensively in production. - #[serde(deserialize_with = "deserialize_optional_duration_from_seconds_ignore_zero")] - pub span_dedup_timeout: Option, - /// @env `DD_API_KEY_SECRET_RELOAD_INTERVAL` - /// - /// The interval at which the Datadog API key is reloaded, in seconds. - /// If None, the API key will not be reloaded. - /// Default is `None`. - #[serde(deserialize_with = "deserialize_optional_duration_from_seconds_ignore_zero")] - pub api_key_secret_reload_interval: Option, - /// @env `DD_SERVERLESS_APPSEC_ENABLED` - /// - /// Enable Application and API Protection (AAP), previously known as AppSec/ASM, for AWS Lambda. - /// Default is `false`. - /// - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub serverless_appsec_enabled: Option, - /// @env `DD_APPSEC_RULES` - /// - /// The path to a user-configured App & API Protection ruleset (in JSON format). - #[serde(deserialize_with = "deserialize_optional_string")] - pub appsec_rules: Option, - /// @env `DD_APPSEC_WAF_TIMEOUT` - /// - /// The timeout for the WAF to process a request, in microseconds. - #[serde(deserialize_with = "deserialize_optional_duration_from_microseconds")] - pub appsec_waf_timeout: Option, - /// @env `DD_API_SECURITY_ENABLED` - /// - /// Enable API Security for AWS Lambda. - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub api_security_enabled: Option, - /// @env `DD_API_SECURITY_SAMPLE_DELAY` - /// - /// The delay between two samples of the API Security schema collection, in seconds. - #[serde(deserialize_with = "deserialize_optional_duration_from_seconds")] - pub api_security_sample_delay: Option, } #[allow(clippy::too_many_lines)] -fn merge_config(config: &mut Config, env_config: &EnvConfig) { +fn merge_config(config: &mut Config, env_config: &EnvConfig) { // Basic fields merge_string!(config, env_config, site); merge_string!(config, env_config, api_key); @@ -654,44 +541,19 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { otlp_config_traces_probabilistic_sampler_sampling_percentage ); merge_option_to_value!(config, env_config, otlp_config_logs_enabled); - - // AWS Lambda - merge_string!(config, env_config, api_key_secret_arn); - merge_string!(config, env_config, kms_api_key); - merge_string!(config, env_config, api_key_ssm_arn); - merge_option_to_value!(config, env_config, serverless_logs_enabled); - - // Handle serverless_logs_enabled with OR logic: if either DD_LOGS_ENABLED or DD_SERVERLESS_LOGS_ENABLED is true, enable logs - if env_config.serverless_logs_enabled.is_some() || env_config.logs_enabled.is_some() { - config.serverless_logs_enabled = env_config.serverless_logs_enabled.unwrap_or(false) - || env_config.logs_enabled.unwrap_or(false); - } - - merge_option_to_value!(config, env_config, serverless_flush_strategy); - merge_option_to_value!(config, env_config, enhanced_metrics); - merge_option_to_value!(config, env_config, lambda_proc_enhanced_metrics); - merge_option_to_value!(config, env_config, capture_lambda_payload); - merge_option_to_value!(config, env_config, capture_lambda_payload_max_depth); - merge_option_to_value!(config, env_config, compute_trace_stats_on_extension); - merge_option!(config, env_config, span_dedup_timeout); - merge_option!(config, env_config, api_key_secret_reload_interval); - merge_option_to_value!(config, env_config, serverless_appsec_enabled); - merge_option!(config, env_config, appsec_rules); - merge_option_to_value!(config, env_config, appsec_waf_timeout); - merge_option_to_value!(config, env_config, api_security_enabled); - merge_option_to_value!(config, env_config, api_security_sample_delay); } #[derive(Debug, PartialEq, Clone, Copy)] #[allow(clippy::module_name_repetitions)] pub struct EnvConfigSource; -impl ConfigSource for EnvConfigSource { - fn load(&self, config: &mut Config) -> Result<(), ConfigError> { +impl ConfigSource for EnvConfigSource { + fn load(&self, config: &mut Config) -> Result<(), ConfigError> { let figment = Figment::new() .merge(Env::prefixed("DATADOG_")) .merge(Env::prefixed("DD_")); + // Extract core config fields match figment.extract::() { Ok(env_config) => merge_config(config, &env_config), Err(e) => { @@ -701,6 +563,16 @@ impl ConfigSource for EnvConfigSource { } } + // Extract extension fields via dual extraction + match figment.extract::() { + Ok(ext_source) => config.ext.merge_from(&ext_source), + Err(e) => { + tracing::warn!( + "Failed to parse extension config from environment variables: {e}, using default extension config." + ); + } + } + Ok(()) } } @@ -709,12 +581,9 @@ impl ConfigSource for EnvConfigSource { #[cfg(test)] #[allow(clippy::result_large_err)] mod tests { - use std::time::Duration; - use super::*; use crate::{ Config, TracePropagationStyle, - flush_strategy::{FlushStrategy, PeriodicStrategy}, log_level::LogLevel, processing_rule::{Kind, ProcessingRule}, }; @@ -728,6 +597,7 @@ mod tests { /// corresponding entry in the arrays below. #[test] #[allow(clippy::too_many_lines)] + #[allow(clippy::field_reassign_with_default)] fn test_all_env_fields_wrong_type_fallback_to_default() { // Non-string fields → invalid values that exercise graceful fallback. let invalid_non_string_env_vars: &[(&str, &str)] = &[ @@ -737,7 +607,6 @@ mod tests { ("DD_LOGS_CONFIG_COMPRESSION_LEVEL", "not_a_number"), ("DD_APM_CONFIG_COMPRESSION_LEVEL", "not_a_number"), ("DD_METRICS_CONFIG_COMPRESSION_LEVEL", "not_a_number"), - ("DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH", "not_a_number"), ("DD_DOGSTATSD_SO_RCVBUF", "not_a_number"), ("DD_DOGSTATSD_BUFFER_SIZE", "not_a_number"), ("DD_DOGSTATSD_QUEUE_SIZE", "not_a_number"), @@ -764,12 +633,6 @@ mod tests { ("DD_TRACE_PROPAGATION_EXTRACT_FIRST", "not_a_bool"), ("DD_TRACE_PROPAGATION_HTTP_BAGGAGE_ENABLED", "not_a_bool"), ("DD_TRACE_AWS_SERVICE_REPRESENTATION_ENABLED", "not_a_bool"), - ("DD_ENHANCED_METRICS", "not_a_bool"), - ("DD_LAMBDA_PROC_ENHANCED_METRICS", "not_a_bool"), - ("DD_CAPTURE_LAMBDA_PAYLOAD", "not_a_bool"), - ("DD_COMPUTE_TRACE_STATS_ON_EXTENSION", "not_a_bool"), - ("DD_SERVERLESS_APPSEC_ENABLED", "not_a_bool"), - ("DD_API_SECURITY_ENABLED", "not_a_bool"), ("DD_OTLP_CONFIG_TRACES_ENABLED", "not_a_bool"), ( "DD_OTLP_CONFIG_TRACES_SPAN_NAME_AS_RESOURCE_NAME", @@ -798,16 +661,8 @@ mod tests { "DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED", "not_a_bool", ), - ("DD_SERVERLESS_LOGS_ENABLED", "not_a_bool"), - ("DD_LOGS_ENABLED", "not_a_bool"), // Enum ("DD_LOG_LEVEL", "invalid_level_999"), - ("DD_SERVERLESS_FLUSH_STRATEGY", "[[[invalid"), - // Duration - ("DD_SPAN_DEDUP_TIMEOUT", "not_a_number"), - ("DD_API_KEY_SECRET_RELOAD_INTERVAL", "not_a_number"), - ("DD_APPSEC_WAF_TIMEOUT", "not_a_number"), - ("DD_API_SECURITY_SAMPLE_DELAY", "not_a_number"), // JSON ("DD_ADDITIONAL_ENDPOINTS", "not_json{{"), ("DD_APM_ADDITIONAL_ENDPOINTS", "not_json{{"), @@ -871,16 +726,6 @@ mod tests { "keep", ), ("DD_OTLP_CONFIG_METRICS_SUMMARIES_MODE", "noquantiles"), - ( - "DD_API_KEY_SECRET_ARN", - "arn:aws:secretsmanager:us-east-1:123:secret:key", - ), - ("DD_KMS_API_KEY", "kms-encrypted-key"), - ( - "DD_API_KEY_SSM_ARN", - "arn:aws:ssm:us-east-1:123:parameter/key", - ), - ("DD_APPSEC_RULES", "/opt/custom-rules.json"), ]; // Programmatic guard: count `pub ` fields in the EnvConfig struct from @@ -913,7 +758,7 @@ mod tests { jail.set_env(key, value); } - let mut config = Config::default(); + let mut config: Config = Config::default(); // This MUST succeed — no single field should crash the whole config EnvConfigSource .load(&mut config) @@ -921,7 +766,7 @@ mod tests { // Build expected: string fields have their non-default values, // all non-string fields stay at defaults. - let mut expected = Config::default(); + let mut expected: Config = Config::default(); // String fields (merge_string! → Config String) expected.site = "custom-site.example.com".to_string(); expected.api_key = "test-api-key-12345".to_string(); @@ -931,10 +776,6 @@ mod tests { expected.observability_pipelines_worker_logs_url = "https://opw.example.com".to_string(); expected.apm_dd_url = "https://custom-apm.example.com".to_string(); - expected.api_key_secret_arn = - "arn:aws:secretsmanager:us-east-1:123:secret:key".to_string(); - expected.kms_api_key = "kms-encrypted-key".to_string(); - expected.api_key_ssm_arn = "arn:aws:ssm:us-east-1:123:parameter/key".to_string(); // Option fields (merge_option! → Config Option) expected.proxy_https = Some("https://proxy.example.com".to_string()); expected.http_protocol = Some("http1".to_string()); @@ -955,7 +796,6 @@ mod tests { expected.otlp_config_metrics_sums_initial_cumulativ_monotonic_value = Some("keep".to_string()); expected.otlp_config_metrics_summaries_mode = Some("noquantiles".to_string()); - expected.appsec_rules = Some("/opt/custom-rules.json".to_string()); assert_eq!(config, expected); Ok(()) @@ -1105,28 +945,7 @@ mod tests { jail.set_env("DD_DOGSTATSD_BUFFER_SIZE", "65507"); jail.set_env("DD_DOGSTATSD_QUEUE_SIZE", "2048"); - // AWS Lambda - jail.set_env( - "DD_API_KEY_SECRET_ARN", - "arn:aws:secretsmanager:region:account:secret:datadog-api-key", - ); - jail.set_env("DD_KMS_API_KEY", "test-kms-key"); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "false"); - jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,60000"); - jail.set_env("DD_ENHANCED_METRICS", "false"); - jail.set_env("DD_LAMBDA_PROC_ENHANCED_METRICS", "false"); - jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "true"); - jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH", "5"); - jail.set_env("DD_COMPUTE_TRACE_STATS_ON_EXTENSION", "true"); - jail.set_env("DD_SPAN_DEDUP_TIMEOUT", "5"); - jail.set_env("DD_API_KEY_SECRET_RELOAD_INTERVAL", "10"); - jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); - jail.set_env("DD_APPSEC_RULES", "/path/to/rules.json"); - jail.set_env("DD_APPSEC_WAF_TIMEOUT", "1000000"); // Microseconds - jail.set_env("DD_API_SECURITY_ENABLED", "0"); // Seconds - jail.set_env("DD_API_SECURITY_SAMPLE_DELAY", "60"); // Seconds - - let mut config = Config::default(); + let mut config: Config = Config::default(); let env_config_source = EnvConfigSource; env_config_source .load(&mut config) @@ -1263,26 +1082,7 @@ mod tests { dogstatsd_so_rcvbuf: Some(1_048_576), dogstatsd_buffer_size: Some(65507), dogstatsd_queue_size: Some(2048), - api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key" - .to_string(), - kms_api_key: "test-kms-key".to_string(), - api_key_ssm_arn: String::default(), - serverless_logs_enabled: false, - serverless_flush_strategy: FlushStrategy::Periodically(PeriodicStrategy { - interval: 60000, - }), - enhanced_metrics: false, - lambda_proc_enhanced_metrics: false, - capture_lambda_payload: true, - capture_lambda_payload_max_depth: 5, - compute_trace_stats_on_extension: true, - span_dedup_timeout: Some(Duration::from_secs(5)), - api_key_secret_reload_interval: Some(Duration::from_secs(10)), - serverless_appsec_enabled: true, - appsec_rules: Some("/path/to/rules.json".to_string()), - appsec_waf_timeout: Duration::from_secs(1), - api_security_enabled: false, - api_security_sample_delay: Duration::from_secs(60), + ext: crate::NoExtension, }; assert_eq!(config, expected_config); @@ -1291,165 +1091,6 @@ mod tests { }); } - #[test] - fn test_dd_logs_enabled_true() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_LOGS_ENABLED", "true"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - assert!(config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_dd_logs_enabled_false() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_LOGS_ENABLED", "false"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - assert!(!config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_dd_serverless_logs_enabled_true() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "true"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - assert!(config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_dd_serverless_logs_enabled_false() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "false"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - assert!(!config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_both_logs_enabled_true() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_LOGS_ENABLED", "true"); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "true"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - assert!(config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_both_logs_enabled_false() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_LOGS_ENABLED", "false"); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "false"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - assert!(!config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_logs_enabled_true_serverless_logs_enabled_false() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_LOGS_ENABLED", "true"); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "false"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - // OR logic: if either is true, logs are enabled - assert!(config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_logs_enabled_false_serverless_logs_enabled_true() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_LOGS_ENABLED", "false"); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "true"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - // OR logic: if either is true, logs are enabled - assert!(config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_neither_logs_enabled_set_uses_default() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - // Default value is true - assert!(config.serverless_logs_enabled); - Ok(()) - }); - } - #[test] fn test_dogstatsd_config_from_env() { figment::Jail::expect_with(|jail| { @@ -1458,7 +1099,7 @@ mod tests { jail.set_env("DD_DOGSTATSD_BUFFER_SIZE", "65507"); jail.set_env("DD_DOGSTATSD_QUEUE_SIZE", "2048"); - let mut config = Config::default(); + let mut config: Config = Config::default(); let env_config_source = EnvConfigSource; env_config_source .load(&mut config) @@ -1476,7 +1117,7 @@ mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); - let mut config = Config::default(); + let mut config: Config = Config::default(); let env_config_source = EnvConfigSource; env_config_source .load(&mut config) diff --git a/crates/datadog-agent-config/src/sources/mod.rs b/crates/datadog-agent-config/src/sources/mod.rs new file mode 100644 index 0000000..dc4d398 --- /dev/null +++ b/crates/datadog-agent-config/src/sources/mod.rs @@ -0,0 +1,2 @@ +pub mod env; +pub mod yaml; diff --git a/crates/datadog-agent-config/yaml.rs b/crates/datadog-agent-config/src/sources/yaml.rs similarity index 85% rename from crates/datadog-agent-config/yaml.rs rename to crates/datadog-agent-config/src/sources/yaml.rs index 2c9d49f..933a9fd 100644 --- a/crates/datadog-agent-config/yaml.rs +++ b/crates/datadog-agent-config/src/sources/yaml.rs @@ -1,15 +1,12 @@ -use std::time::Duration; use std::{collections::HashMap, path::PathBuf}; use crate::{ - Config, ConfigError, ConfigSource, ProcessingRule, TracePropagationStyle, + Config, ConfigError, ConfigExtension, ConfigSource, ProcessingRule, TracePropagationStyle, additional_endpoints::deserialize_additional_endpoints, deserialize_apm_replace_rules, deserialize_key_value_pair_array_to_hashmap, deserialize_option_lossless, - deserialize_optional_bool_from_anything, deserialize_optional_duration_from_microseconds, - deserialize_optional_duration_from_seconds, - deserialize_optional_duration_from_seconds_ignore_zero, deserialize_optional_string, + deserialize_optional_bool_from_anything, deserialize_optional_string, deserialize_processing_rules, deserialize_string_or_int, deserialize_trace_propagation_style, - deserialize_with_default, flush_strategy::FlushStrategy, log_level::LogLevel, + deserialize_with_default, log_level::LogLevel, logs_additional_endpoints::LogsAdditionalEndpoint, merge_hashmap, merge_option, merge_option_to_value, merge_string, merge_vec, service_mapping::deserialize_service_mapping, }; @@ -108,40 +105,6 @@ pub struct YamlConfig { // OTLP #[serde(deserialize_with = "deserialize_with_default")] pub otlp_config: Option, - - // AWS Lambda - #[serde(deserialize_with = "deserialize_optional_string")] - pub api_key_secret_arn: Option, - #[serde(deserialize_with = "deserialize_optional_string")] - pub kms_api_key: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub serverless_logs_enabled: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub logs_enabled: Option, - #[serde(deserialize_with = "deserialize_with_default")] - pub serverless_flush_strategy: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub enhanced_metrics: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub lambda_proc_enhanced_metrics: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub capture_lambda_payload: Option, - #[serde(deserialize_with = "deserialize_option_lossless")] - pub capture_lambda_payload_max_depth: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub compute_trace_stats_on_extension: Option, - #[serde(deserialize_with = "deserialize_optional_duration_from_seconds_ignore_zero")] - pub api_key_secret_reload_interval: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub serverless_appsec_enabled: Option, - #[serde(deserialize_with = "deserialize_optional_string")] - pub appsec_rules: Option, - #[serde(deserialize_with = "deserialize_optional_duration_from_microseconds")] - pub appsec_waf_timeout: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub api_security_enabled: Option, - #[serde(deserialize_with = "deserialize_optional_duration_from_seconds")] - pub api_security_sample_delay: Option, } /// Proxy Config @@ -443,7 +406,7 @@ impl OtlpConfig { } #[allow(clippy::too_many_lines)] -fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { +fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { // Basic fields merge_string!(config, yaml_config, site); merge_string!(config, yaml_config, api_key); @@ -720,29 +683,6 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { merge_option_to_value!(config, otlp_config_logs_enabled, logs, enabled); } } - - // AWS Lambda - merge_string!(config, yaml_config, api_key_secret_arn); - merge_string!(config, yaml_config, kms_api_key); - - // Handle serverless_logs_enabled with OR logic: if either logs_enabled or serverless_logs_enabled is true, enable logs - if yaml_config.serverless_logs_enabled.is_some() || yaml_config.logs_enabled.is_some() { - config.serverless_logs_enabled = yaml_config.serverless_logs_enabled.unwrap_or(false) - || yaml_config.logs_enabled.unwrap_or(false); - } - - merge_option_to_value!(config, yaml_config, serverless_flush_strategy); - merge_option_to_value!(config, yaml_config, enhanced_metrics); - merge_option_to_value!(config, yaml_config, lambda_proc_enhanced_metrics); - merge_option_to_value!(config, yaml_config, capture_lambda_payload); - merge_option_to_value!(config, yaml_config, capture_lambda_payload_max_depth); - merge_option_to_value!(config, yaml_config, compute_trace_stats_on_extension); - merge_option!(config, yaml_config, api_key_secret_reload_interval); - merge_option_to_value!(config, yaml_config, serverless_appsec_enabled); - merge_option!(config, yaml_config, appsec_rules); - merge_option_to_value!(config, yaml_config, appsec_waf_timeout); - merge_option_to_value!(config, yaml_config, api_security_enabled); - merge_option_to_value!(config, yaml_config, api_security_sample_delay); } #[derive(Debug, PartialEq, Clone)] @@ -751,8 +691,8 @@ pub struct YamlConfigSource { pub path: PathBuf, } -impl ConfigSource for YamlConfigSource { - fn load(&self, config: &mut Config) -> Result<(), ConfigError> { +impl ConfigSource for YamlConfigSource { + fn load(&self, config: &mut Config) -> Result<(), ConfigError> { let figment = Figment::new().merge(Yaml::file(self.path.clone())); match figment.extract::() { @@ -764,6 +704,16 @@ impl ConfigSource for YamlConfigSource { } } + // Extract extension fields via dual extraction + match figment.extract::() { + Ok(ext_source) => config.ext.merge_from(&ext_source), + Err(e) => { + tracing::warn!( + "Failed to parse extension config from yaml file: {e}, using default extension config." + ); + } + } + Ok(()) } } @@ -773,9 +723,8 @@ impl ConfigSource for YamlConfigSource { #[allow(clippy::result_large_err)] mod tests { use std::path::Path; - use std::time::Duration; - use crate::{flush_strategy::PeriodicStrategy, log_level::LogLevel, processing_rule::Kind}; + use crate::{log_level::LogLevel, processing_rule::Kind}; use super::*; @@ -785,6 +734,7 @@ mod tests { /// When adding a new field to YamlConfig or any nested struct, add an entry /// here with the wrong type to ensure graceful deserialization is in place. #[test] + #[allow(clippy::field_reassign_with_default)] fn test_all_yaml_fields_wrong_type_fallback_to_default() { figment::Jail::expect_with(|jail| { jail.clear_env(); @@ -891,28 +841,10 @@ otlp_config: mode: "noquantiles" logs: enabled: [1, 2, 3] - -# AWS Lambda -api_key_secret_arn: "arn:aws:secretsmanager:us-east-1:123:secret:key" -kms_api_key: "kms-encrypted-key" -serverless_logs_enabled: [1, 2, 3] -logs_enabled: [1, 2, 3] -serverless_flush_strategy: [1, 2, 3] -enhanced_metrics: [1, 2, 3] -lambda_proc_enhanced_metrics: [1, 2, 3] -capture_lambda_payload: [1, 2, 3] -capture_lambda_payload_max_depth: [1, 2, 3] -compute_trace_stats_on_extension: [1, 2, 3] -api_key_secret_reload_interval: [1, 2, 3] -serverless_appsec_enabled: [1, 2, 3] -appsec_rules: "/opt/custom-rules.json" -appsec_waf_timeout: [1, 2, 3] -api_security_enabled: [1, 2, 3] -api_security_sample_delay: [1, 2, 3] "#, )?; - let mut config = Config::default(); + let mut config: Config = Config::default(); let source = YamlConfigSource { path: PathBuf::from("datadog.yaml"), }; @@ -923,15 +855,12 @@ api_security_sample_delay: [1, 2, 3] // Build expected: string fields have their non-default values, // all non-string fields stay at defaults. - let mut expected = Config::default(); + let mut expected: Config = Config::default(); expected.site = "custom-site.example.com".to_string(); expected.api_key = "test-api-key-12345".to_string(); expected.dd_url = "https://custom-metrics.example.com".to_string(); expected.logs_config_logs_dd_url = "https://custom-logs.example.com".to_string(); expected.apm_dd_url = "https://custom-apm.example.com".to_string(); - expected.api_key_secret_arn = - "arn:aws:secretsmanager:us-east-1:123:secret:key".to_string(); - expected.kms_api_key = "kms-encrypted-key".to_string(); // Option fields expected.proxy_https = Some("https://proxy.example.com".to_string()); expected.http_protocol = Some("http1".to_string()); @@ -951,7 +880,6 @@ api_security_sample_delay: [1, 2, 3] expected.otlp_config_metrics_sums_initial_cumulativ_monotonic_value = Some("keep".to_string()); expected.otlp_config_metrics_summaries_mode = Some("noquantiles".to_string()); - expected.appsec_rules = Some("/opt/custom-rules.json".to_string()); assert_eq!(config, expected); Ok(()) @@ -1082,27 +1010,10 @@ otlp_config: mode: "quantiles" logs: enabled: true - -# AWS Lambda -api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key" -kms_api_key: "test-kms-key" -serverless_logs_enabled: false -serverless_flush_strategy: "periodically,60000" -enhanced_metrics: false -lambda_proc_enhanced_metrics: false -capture_lambda_payload: true -capture_lambda_payload_max_depth: 5 -compute_trace_stats_on_extension: true -api_key_secret_reload_interval: 0 -serverless_appsec_enabled: true -appsec_rules: "/path/to/rules.json" -appsec_waf_timeout: 1000000 # Microseconds -api_security_enabled: false -api_security_sample_delay: 60 # Seconds "#, )?; - let mut config = Config::default(); + let mut config: Config = Config::default(); let yaml_config_source = YamlConfigSource { path: Path::new("datadog.yaml").to_path_buf(), }; @@ -1216,28 +1127,6 @@ api_security_sample_delay: 60 # Seconds otlp_config_metrics_summaries_mode: Some("quantiles".to_string()), otlp_config_traces_probabilistic_sampler_sampling_percentage: Some(50), otlp_config_logs_enabled: true, - api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key" - .to_string(), - kms_api_key: "test-kms-key".to_string(), - api_key_ssm_arn: String::default(), - serverless_logs_enabled: false, - serverless_flush_strategy: FlushStrategy::Periodically(PeriodicStrategy { - interval: 60000, - }), - enhanced_metrics: false, - lambda_proc_enhanced_metrics: false, - capture_lambda_payload: true, - capture_lambda_payload_max_depth: 5, - compute_trace_stats_on_extension: true, - span_dedup_timeout: None, - api_key_secret_reload_interval: None, - - serverless_appsec_enabled: true, - appsec_rules: Some("/path/to/rules.json".to_string()), - appsec_waf_timeout: Duration::from_secs(1), - api_security_enabled: false, - api_security_sample_delay: Duration::from_secs(60), - apm_filter_tags_require: None, apm_filter_tags_reject: None, apm_filter_tags_regex_require: None, @@ -1246,6 +1135,7 @@ api_security_sample_delay: 60 # Seconds dogstatsd_so_rcvbuf: Some(1_048_576), dogstatsd_buffer_size: Some(65507), dogstatsd_queue_size: Some(2048), + ext: crate::NoExtension, }; // Assert that @@ -1267,7 +1157,7 @@ dogstatsd_buffer_size: 16384 dogstatsd_queue_size: 512 ", )?; - let mut config = Config::default(); + let mut config: Config = Config::default(); let yaml_config_source = YamlConfigSource { path: Path::new("datadog.yaml").to_path_buf(), }; @@ -1287,7 +1177,7 @@ dogstatsd_queue_size: 512 figment::Jail::expect_with(|jail| { jail.clear_env(); jail.create_file("datadog.yaml", "")?; - let mut config = Config::default(); + let mut config: Config = Config::default(); let yaml_config_source = YamlConfigSource { path: Path::new("datadog.yaml").to_path_buf(), }; From 9314f6c10542909427d97432da1de6ba90241972 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Mon, 20 Apr 2026 08:39:42 -0400 Subject: [PATCH 08/10] chore: fix cargo audit errors and warnings, cargo clippy warnings (#117) * update crates for cargo audit * apply clippy fixes * apply formatting, update license --- Cargo.lock | 139 +++++++------------ LICENSE-3rdparty.csv | 3 - crates/datadog-trace-agent/Cargo.toml | 2 +- crates/datadog-trace-agent/src/mini_agent.rs | 16 +-- 4 files changed, 59 insertions(+), 101 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8970e45..1589fac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,7 +89,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -179,7 +179,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.117", + "syn", ] [[package]] @@ -509,7 +509,7 @@ dependencies = [ "opentelemetry", "opentelemetry-semantic-conventions", "opentelemetry_sdk", - "rand 0.8.5", + "rand 0.8.6", "rustc_version_runtime", "serde", "serde_json", @@ -608,7 +608,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "unicode-xid", ] @@ -630,7 +630,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -668,12 +668,13 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "duplicate" -version = "0.4.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0a4be4cd710e92098de6ad258e6e7c24af11c29c5142f3c6f2a545652480ff8" +checksum = "8e92f10a49176cbffacaedabfaa11d51db1ea0f80a83c26e1873b43cd1742c24" dependencies = [ - "heck 0.4.1", - "proc-macro-error", + "heck", + "proc-macro2", + "proc-macro2-diagnostics", ] [[package]] @@ -872,7 +873,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -1037,12 +1038,6 @@ dependencies = [ "http", ] -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -1678,7 +1673,7 @@ dependencies = [ "libdd-trace-normalization 1.0.2", "libdd-trace-protobuf 2.0.0", "prost 0.14.3", - "rand 0.8.5", + "rand 0.8.6", "rmp", "rmp-serde", "rmpv", @@ -1710,7 +1705,7 @@ dependencies = [ "libdd-trace-normalization 1.0.3", "libdd-trace-protobuf 3.0.0", "prost 0.14.3", - "rand 0.8.5", + "rand 0.8.6", "rmp", "rmp-serde", "rmpv", @@ -1844,7 +1839,7 @@ dependencies = [ "hyper-util", "log", "pin-project-lite", - "rand 0.9.2", + "rand 0.9.4", "regex", "serde_json", "serde_urlencoded", @@ -1940,7 +1935,7 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand 0.9.2", + "rand 0.9.4", "thiserror 2.0.18", ] @@ -2020,7 +2015,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2056,7 +2051,7 @@ checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2102,31 +2097,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.117", -] - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", + "syn", ] [[package]] @@ -2146,7 +2117,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "version_check", "yansi", ] @@ -2161,7 +2132,7 @@ dependencies = [ "bit-vec", "bitflags", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", @@ -2196,7 +2167,7 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.5.0", + "heck", "itertools 0.14.0", "log", "multimap", @@ -2206,7 +2177,7 @@ dependencies = [ "prost 0.13.5", "prost-types", "regex", - "syn 2.0.117", + "syn", "tempfile", ] @@ -2220,7 +2191,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2233,7 +2204,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2332,7 +2303,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -2381,9 +2352,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -2392,9 +2363,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -2473,7 +2444,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2676,9 +2647,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ "aws-lc-rs", "ring", @@ -2816,7 +2787,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2890,7 +2861,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2998,16 +2969,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.117" @@ -3036,7 +2997,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3106,7 +3067,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3117,7 +3078,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3178,7 +3139,7 @@ checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3247,7 +3208,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3315,7 +3276,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3375,7 +3336,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad06847b7afb65c7866a36664b75c40b895e318cea4f71299f013fb22965329d" dependencies = [ "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3582,7 +3543,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wasm-bindgen-shared", ] @@ -3941,7 +3902,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" dependencies = [ "anyhow", - "heck 0.5.0", + "heck", "wit-parser", ] @@ -3952,10 +3913,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", - "heck 0.5.0", + "heck", "indexmap", "prettyplease", - "syn 2.0.117", + "syn", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -3971,7 +3932,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -4044,7 +4005,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -4065,7 +4026,7 @@ checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4085,7 +4046,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -4125,7 +4086,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 2d04b2a..7e32f63 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -84,7 +84,6 @@ hashbrown,https://github.com/rust-lang/hashbrown,MIT OR Apache-2.0,Amanieu d'Ant headers,https://github.com/hyperium/headers,MIT,Sean McArthur headers-core,https://github.com/hyperium/headers,MIT,Sean McArthur heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,The heck Authors -heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,Without Boats hex,https://github.com/KokaKiwi/rust-hex,MIT OR Apache-2.0,KokaKiwi home,https://github.com/rust-lang/cargo,MIT OR Apache-2.0,Brian Anderson http,https://github.com/hyperium/http,MIT OR Apache-2.0,"Alex Crichton , Carl Lerche , Sean McArthur " @@ -168,8 +167,6 @@ pin-utils,https://github.com/rust-lang-nursery/pin-utils,MIT OR Apache-2.0,Josef potential_utf,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers ppv-lite86,https://github.com/cryptocorrosion/cryptocorrosion,MIT OR Apache-2.0,The CryptoCorrosion Contributors prettyplease,https://github.com/dtolnay/prettyplease,MIT OR Apache-2.0,David Tolnay -proc-macro-error,https://gitlab.com/CreepySkeleton/proc-macro-error,MIT OR Apache-2.0,CreepySkeleton -proc-macro-error-attr,https://gitlab.com/CreepySkeleton/proc-macro-error,MIT OR Apache-2.0,CreepySkeleton proc-macro2,https://github.com/dtolnay/proc-macro2,MIT OR Apache-2.0,"David Tolnay , Alex Crichton " proc-macro2-diagnostics,https://github.com/SergioBenitez/proc-macro2-diagnostics,MIT OR Apache-2.0,Sergio Benitez prost,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " diff --git a/crates/datadog-trace-agent/Cargo.toml b/crates/datadog-trace-agent/Cargo.toml index 1a69733..69eb85f 100644 --- a/crates/datadog-trace-agent/Cargo.toml +++ b/crates/datadog-trace-agent/Cargo.toml @@ -37,7 +37,7 @@ bytes = "1.10.1" [dev-dependencies] rmp-serde = "1.1.1" serial_test = "2.0.0" -duplicate = "0.4.1" +duplicate = "2.0.1" temp-env = "0.3.6" tempfile = "3.3.0" libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad", features = [ diff --git a/crates/datadog-trace-agent/src/mini_agent.rs b/crates/datadog-trace-agent/src/mini_agent.rs index 7fa5a92..ae07481 100644 --- a/crates/datadog-trace-agent/src/mini_agent.rs +++ b/crates/datadog-trace-agent/src/mini_agent.rs @@ -11,7 +11,7 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Instant; use tokio::sync::mpsc::{self, Receiver, Sender}; -use tracing::{debug, error, warn}; +use tracing::{debug, error}; use crate::http_utils::{log_and_create_http_response, verify_request_content_length}; use crate::proxy_flusher::{ProxyFlusher, ProxyRequest}; @@ -191,14 +191,14 @@ impl MiniAgent { let sentinel = std::path::Path::new(LAMBDA_LITE_SENTINEL_PATH); // SAFETY: LAMBDA_LITE_SENTINEL_PATH is a hard-coded absolute path, // so .parent() always returns Some. - if let Some(parent) = sentinel.parent() { - if let Err(e) = tokio::fs::create_dir_all(parent).await { - error!( - "Could not create parent directory for Lambda Lite sentinel \ + if let Some(parent) = sentinel.parent() + && let Err(e) = tokio::fs::create_dir_all(parent).await + { + error!( + "Could not create parent directory for Lambda Lite sentinel \ file at {}: {}.", - LAMBDA_LITE_SENTINEL_PATH, e - ); - } + LAMBDA_LITE_SENTINEL_PATH, e + ); } if let Err(e) = tokio::fs::write(sentinel, b"").await { error!( From 8b229c13e41b4932b595d2513154829da0354eb9 Mon Sep 17 00:00:00 2001 From: Kathie Huang <46662481+kathiehuang@users.noreply.github.com> Date: Mon, 20 Apr 2026 17:08:23 -0400 Subject: [PATCH 09/10] [SVLS-8799] Update libdatadog revision (#115) * Update libdatadog rev * Update rustls-native-certs * Update stats_flusher to use DefaultHttpClient * Adapt trace flusher to the new trait-based API * Add FUNCTION_TARGET to unit tests, update license, update rustls-webpki --- Cargo.lock | 84 +++++++++++------ LICENSE-3rdparty.csv | 4 +- crates/datadog-agent-config/Cargo.toml | 4 +- crates/datadog-fips/Cargo.toml | 2 +- crates/datadog-serverless-compat/Cargo.toml | 2 +- crates/datadog-trace-agent/Cargo.toml | 11 +-- crates/datadog-trace-agent/src/config.rs | 4 + .../datadog-trace-agent/src/stats_flusher.rs | 3 +- .../datadog-trace-agent/src/trace_flusher.rs | 90 +++++++++++++------ 9 files changed, 140 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1589fac..2df7122 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -448,7 +448,7 @@ dependencies = [ "dogstatsd", "figment", "libdd-trace-obfuscation", - "libdd-trace-utils 3.0.0", + "libdd-trace-utils 3.0.1", "log", "serde", "serde-aux", @@ -542,7 +542,7 @@ dependencies = [ "datadog-logs-agent", "datadog-trace-agent", "dogstatsd", - "libdd-trace-utils 3.0.0", + "libdd-trace-utils 3.0.1", "reqwest", "serde_json", "tokio", @@ -565,10 +565,11 @@ dependencies = [ "hyper", "hyper-http-proxy", "hyper-util", - "libdd-common 3.0.1", + "libdd-capabilities", + "libdd-common 3.0.2", "libdd-trace-obfuscation", - "libdd-trace-protobuf 3.0.0", - "libdd-trace-utils 3.0.0", + "libdd-trace-protobuf 3.0.1", + "libdd-trace-utils 3.0.1", "reqwest", "rmp-serde", "serde", @@ -1426,6 +1427,28 @@ version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +[[package]] +name = "libdd-capabilities" +version = "0.1.0" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" +dependencies = [ + "anyhow", + "bytes", + "http", + "thiserror 1.0.69", +] + +[[package]] +name = "libdd-capabilities-impl" +version = "0.1.0" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" +dependencies = [ + "bytes", + "http", + "libdd-capabilities", + "libdd-common 3.0.2", +] + [[package]] name = "libdd-common" version = "2.0.1" @@ -1459,8 +1482,8 @@ dependencies = [ [[package]] name = "libdd-common" -version = "3.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +version = "3.0.2" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" dependencies = [ "anyhow", "bytes", @@ -1477,6 +1500,7 @@ dependencies = [ "hyper-rustls", "hyper-util", "libc", + "libdd-capabilities", "nix", "pin-project", "regex", @@ -1581,7 +1605,7 @@ dependencies = [ [[package]] name = "libdd-tinybytes" version = "1.1.0" -source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" dependencies = [ "serde", ] @@ -1598,23 +1622,23 @@ dependencies = [ [[package]] name = "libdd-trace-normalization" -version = "1.0.3" -source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +version = "2.0.0" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" dependencies = [ "anyhow", - "libdd-trace-protobuf 3.0.0", + "libdd-trace-protobuf 3.0.1", ] [[package]] name = "libdd-trace-obfuscation" -version = "1.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +version = "2.0.0" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" dependencies = [ "anyhow", "fluent-uri", - "libdd-common 3.0.1", - "libdd-trace-protobuf 3.0.0", - "libdd-trace-utils 3.0.0", + "libdd-common 3.0.2", + "libdd-trace-protobuf 3.0.1", + "libdd-trace-utils 3.0.1", "log", "percent-encoding", "regex", @@ -1635,8 +1659,8 @@ dependencies = [ [[package]] name = "libdd-trace-protobuf" -version = "3.0.0" -source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +version = "3.0.1" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" dependencies = [ "prost 0.14.3", "serde", @@ -1685,25 +1709,29 @@ dependencies = [ [[package]] name = "libdd-trace-utils" -version = "3.0.0" -source = "git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad#8c88979985154d6d97c0fc2ca9039682981eacad" +version = "3.0.1" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" dependencies = [ "anyhow", + "base64 0.22.1", "bytes", "cargo-platform", "cargo_metadata", "flate2", "futures", + "getrandom 0.2.17", "http", "http-body", "http-body-util", "httpmock", "hyper", "indexmap", - "libdd-common 3.0.1", - "libdd-tinybytes 1.1.0 (git+https://github.com/DataDog/libdatadog?rev=8c88979985154d6d97c0fc2ca9039682981eacad)", - "libdd-trace-normalization 1.0.3", - "libdd-trace-protobuf 3.0.0", + "libdd-capabilities", + "libdd-capabilities-impl", + "libdd-common 3.0.2", + "libdd-tinybytes 1.1.0 (git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665)", + "libdd-trace-normalization 2.0.0", + "libdd-trace-protobuf 3.0.1", "prost 0.14.3", "rand 0.8.6", "rmp", @@ -1901,9 +1929,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openssl-probe" -version = "0.2.1" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "opentelemetry" @@ -2625,9 +2653,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" dependencies = [ "openssl-probe", "rustls-pki-types", diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 7e32f63..c89c5e2 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -117,6 +117,8 @@ js-sys,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/js-sys,MI lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2.0,Marvin Löbel leb128fmt,https://github.com/bluk/leb128fmt,MIT OR Apache-2.0,Bryant Luk libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers +libdd-capabilities,https://github.com/DataDog/libdatadog/tree/main/libdd-capabilities,Apache-2.0,The libdd-capabilities Authors +libdd-capabilities-impl,https://github.com/DataDog/libdatadog/tree/main/libdd-capabilities-impl,Apache-2.0,The libdd-capabilities-impl Authors libdd-common,https://github.com/DataDog/libdatadog/tree/main/datadog-common,Apache-2.0,The libdd-common Authors libdd-data-pipeline,https://github.com/DataDog/libdatadog/tree/main/libdd-data-pipeline,Apache-2.0,The libdd-data-pipeline Authors libdd-ddsketch,https://github.com/DataDog/libdatadog/tree/main/libdd-ddsketch,Apache-2.0,The libdd-ddsketch Authors @@ -147,7 +149,7 @@ nom,https://github.com/Geal/nom,MIT,contact@geoffroycouprie.com nu-ansi-term,https://github.com/nushell/nu-ansi-term,MIT,"ogham@bsago.me, Ryan Scheel (Havvy) , Josh Triplett , The Nushell Project Developers" num-traits,https://github.com/rust-num/num-traits,MIT OR Apache-2.0,The Rust Project Developers once_cell,https://github.com/matklad/once_cell,MIT OR Apache-2.0,Aleksey Kladov -openssl-probe,https://github.com/rustls/openssl-probe,MIT OR Apache-2.0,Alex Crichton +openssl-probe,https://github.com/alexcrichton/openssl-probe,MIT OR Apache-2.0,Alex Crichton opentelemetry,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry,Apache-2.0,The opentelemetry Authors opentelemetry-semantic-conventions,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-semantic-conventions,Apache-2.0,The opentelemetry-semantic-conventions Authors opentelemetry_sdk,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-sdk,Apache-2.0,The opentelemetry_sdk Authors diff --git a/crates/datadog-agent-config/Cargo.toml b/crates/datadog-agent-config/Cargo.toml index b9477ac..bd87bcb 100644 --- a/crates/datadog-agent-config/Cargo.toml +++ b/crates/datadog-agent-config/Cargo.toml @@ -6,8 +6,8 @@ license.workspace = true [dependencies] figment = { version = "0.10", default-features = false, features = ["yaml", "env"] } -libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } +libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } log = { version = "0.4", default-features = false } serde = { version = "1.0", default-features = false, features = ["derive"] } serde-aux = { version = "4.7", default-features = false } diff --git a/crates/datadog-fips/Cargo.toml b/crates/datadog-fips/Cargo.toml index 9758f41..e09a358 100644 --- a/crates/datadog-fips/Cargo.toml +++ b/crates/datadog-fips/Cargo.toml @@ -9,7 +9,7 @@ repository.workspace = true [dependencies] reqwest = { version = "0.12.4", features = ["json", "http2"], default-features = false } rustls = { version = "0.23.18", default-features = false, features = ["fips"], optional = true } -rustls-native-certs = { version = "0.8.1", optional = true } +rustls-native-certs = { version = ">=0.8.1, <0.8.3", optional = true } tracing = { version = "0.1.40", default-features = false } [features] diff --git a/crates/datadog-serverless-compat/Cargo.toml b/crates/datadog-serverless-compat/Cargo.toml index b84bb15..6d99940 100644 --- a/crates/datadog-serverless-compat/Cargo.toml +++ b/crates/datadog-serverless-compat/Cargo.toml @@ -12,7 +12,7 @@ windows-pipes = ["datadog-trace-agent/windows-pipes", "dogstatsd/windows-pipes"] [dependencies] datadog-logs-agent = { path = "../datadog-logs-agent" } datadog-trace-agent = { path = "../datadog-trace-agent" } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } datadog-fips = { path = "../datadog-fips", default-features = false } dogstatsd = { path = "../dogstatsd", default-features = true } reqwest = { version = "0.12.4", default-features = false } diff --git a/crates/datadog-trace-agent/Cargo.toml b/crates/datadog-trace-agent/Cargo.toml index 69eb85f..c5d059d 100644 --- a/crates/datadog-trace-agent/Cargo.toml +++ b/crates/datadog-trace-agent/Cargo.toml @@ -24,12 +24,13 @@ async-trait = "0.1.64" tracing = { version = "0.1", default-features = false } serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0" -libdd-common = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } -libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad", features = [ +libdd-capabilities = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } +libdd-common = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } +libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665", features = [ "mini_agent", ] } -libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad" } +libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } datadog-fips = { path = "../datadog-fips" } reqwest = { version = "0.12.23", features = ["json", "http2"], default-features = false } bytes = "1.10.1" @@ -40,6 +41,6 @@ serial_test = "2.0.0" duplicate = "2.0.1" temp-env = "0.3.6" tempfile = "3.3.0" -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8c88979985154d6d97c0fc2ca9039682981eacad", features = [ +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665", features = [ "test-utils", ] } diff --git a/crates/datadog-trace-agent/src/config.rs b/crates/datadog-trace-agent/src/config.rs index 5a7b8a8..ddbd820 100644 --- a/crates/datadog-trace-agent/src/config.rs +++ b/crates/datadog-trace-agent/src/config.rs @@ -296,6 +296,7 @@ mod tests { [ ("DD_API_KEY", Some("_not_a_real_key_")), ("K_SERVICE", Some("function_name")), + ("FUNCTION_TARGET", Some("function_target")), ], || { let config_res = config::Config::new(); @@ -329,6 +330,7 @@ mod tests { [ ("DD_API_KEY", Some("_not_a_real_key_")), ("K_SERVICE", Some("function_name")), + ("FUNCTION_TARGET", Some("function_target")), ("DD_SITE", Some(dd_site)), ], || { @@ -356,6 +358,7 @@ mod tests { [ ("DD_API_KEY", Some("_not_a_real_key_")), ("K_SERVICE", Some("function_name")), + ("FUNCTION_TARGET", Some("function_target")), ("DD_SITE", Some(dd_site)), ], || { @@ -374,6 +377,7 @@ mod tests { [ ("DD_API_KEY", Some("_not_a_real_key_")), ("K_SERVICE", Some("function_name")), + ("FUNCTION_TARGET", Some("function_target")), ("DD_APM_DD_URL", Some("http://127.0.0.1:3333")), ], || { diff --git a/crates/datadog-trace-agent/src/stats_flusher.rs b/crates/datadog-trace-agent/src/stats_flusher.rs index 6c6e580..3bee237 100644 --- a/crates/datadog-trace-agent/src/stats_flusher.rs +++ b/crates/datadog-trace-agent/src/stats_flusher.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use async_trait::async_trait; +use libdd_common::DefaultHttpClient; use std::{sync::Arc, time}; use tokio::sync::{Mutex, mpsc::Receiver}; use tracing::{debug, error}; @@ -76,7 +77,7 @@ impl StatsFlusher for ServerlessStatsFlusher { }; #[allow(clippy::unwrap_used)] - match stats_utils::send_stats_payload( + match stats_utils::send_stats_payload::( serialized_stats_payload, &config.trace_stats_intake, config.trace_stats_intake.api_key.as_ref().unwrap(), diff --git a/crates/datadog-trace-agent/src/trace_flusher.rs b/crates/datadog-trace-agent/src/trace_flusher.rs index 9efebac..638e3fd 100644 --- a/crates/datadog-trace-agent/src/trace_flusher.rs +++ b/crates/datadog-trace-agent/src/trace_flusher.rs @@ -6,7 +6,11 @@ use std::{error::Error, sync::Arc, time}; use tokio::sync::{Mutex, mpsc::Receiver}; use tracing::{debug, error}; -use libdd_common::{GenericHttpClient, http_common}; +use http_body_util::BodyExt; +use libdd_capabilities::http::{HttpClientTrait, HttpError}; +use libdd_capabilities::{MaybeSend, Request, Response}; +use libdd_common::connector::Connector; +use libdd_common::http_common::{self, Body, GenericHttpClient}; use libdd_trace_utils::trace_utils; use libdd_trace_utils::trace_utils::SendData; @@ -75,14 +79,13 @@ impl TraceFlusher for ServerlessTraceFlusher { } debug!("Flushing {} traces", traces.len()); - let http_client = - match ServerlessTraceFlusher::get_http_client(self.config.proxy_url.as_ref()) { - Ok(client) => client, - Err(e) => { - error!("Failed to create HTTP client: {e:?}"); - return; - } - }; + let http_client = match ProxyHttpClient::with_proxy(self.config.proxy_url.as_ref()) { + Ok(client) => client, + Err(e) => { + error!("Failed to create HTTP client: {e:?}"); + return; + } + }; // Retries are handled internally by SendData::send() for coalesced_traces in trace_utils::coalesce_send_data(traces) { @@ -97,26 +100,63 @@ impl TraceFlusher for ServerlessTraceFlusher { } } -impl ServerlessTraceFlusher { - fn get_http_client( - proxy_https: Option<&String>, - ) -> Result< - GenericHttpClient>, - Box, - > { +#[derive(Clone)] +struct ProxyHttpClient { + client: GenericHttpClient>, +} + +impl std::fmt::Debug for ProxyHttpClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProxyHttpClient").finish() + } +} + +impl ProxyHttpClient { + // `HttpClientTrait::new_client` takes no arguments, so we use `with_proxy` to + // take in the proxy URL and build the client. `new_client` is never called on our code path. + fn with_proxy(proxy_https: Option<&String>) -> Result> { if let Some(proxy) = proxy_https { let proxy = hyper_http_proxy::Proxy::new(hyper_http_proxy::Intercept::Https, proxy.parse()?); - let proxy_connector = hyper_http_proxy::ProxyConnector::from_proxy( - libdd_common::connector::Connector::default(), - proxy, - )?; - Ok(http_common::client_builder().build(proxy_connector)) + let proxy_connector = + hyper_http_proxy::ProxyConnector::from_proxy(Connector::default(), proxy)?; + Ok(Self { + client: http_common::client_builder().build(proxy_connector), + }) } else { - let proxy_connector = hyper_http_proxy::ProxyConnector::new( - libdd_common::connector::Connector::default(), - )?; - Ok(http_common::client_builder().build(proxy_connector)) + let proxy_connector = hyper_http_proxy::ProxyConnector::new(Connector::default())?; + Ok(Self { + client: http_common::client_builder().build(proxy_connector), + }) + } + } +} + +impl HttpClientTrait for ProxyHttpClient { + #[allow(clippy::expect_used)] + fn new_client() -> Self { + Self::with_proxy(None).expect("building proxy connector with default TLS should not fail") + } + + fn request( + &self, + req: Request, + ) -> impl std::future::Future, HttpError>> + MaybeSend + { + let client = self.client.clone(); + async move { + let hyper_req = req.map(Body::from_bytes); + let response = client + .request(hyper_req) + .await + .map_err(|e| HttpError::Network(e.into()))?; + let (parts, body) = response.into_parts(); + let collected = body + .collect() + .await + .map_err(|e| HttpError::ResponseBody(e.into()))? + .to_bytes(); + Ok(Response::from_parts(parts, collected)) } } } From c204acd2e20ccefd718b48b8056b32e5f106ad01 Mon Sep 17 00:00:00 2001 From: "dd-octo-sts[bot]" <200755185+dd-octo-sts[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2026 13:09:14 +0000 Subject: [PATCH 10/10] chore(deps): update libdd-common digest to 950562b --- crates/datadog-trace-agent/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/datadog-trace-agent/Cargo.toml b/crates/datadog-trace-agent/Cargo.toml index c5d059d..d057984 100644 --- a/crates/datadog-trace-agent/Cargo.toml +++ b/crates/datadog-trace-agent/Cargo.toml @@ -25,7 +25,7 @@ tracing = { version = "0.1", default-features = false } serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0" libdd-capabilities = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } -libdd-common = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } +libdd-common = { git = "https://github.com/DataDog/libdatadog", rev = "950562bb191205bf16edbb4296e4a8ae33da194c" } libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665", features = [ "mini_agent",