Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 114 additions & 36 deletions dragonfly-client-util/src/types/redacted.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
use dragonfly_api::common::v2::{Download, Hdfs, HuggingFace, ModelScope, ObjectStorage, OpenCsg};
use dragonfly_api::dfdaemon::v2::DownloadPersistentTaskRequest;
use http::header::{HOST, RANGE, USER_AGENT};
use std::collections::HashMap;
use http::Request;
use std::fmt;

/// Placeholder inserted in place of redacted secret material.
Expand Down Expand Up @@ -120,41 +120,6 @@ impl fmt::Debug for RedactedDownload<'_> {
}
}

/// Clones the optional sub-message only when present and scrubs its secrets,
/// so the common absent case costs nothing.
fn scrubbed<T: Clone>(value: &Option<T>, scrub: fn(&mut Option<T>)) -> Option<T> {
let mut value = value.clone();
scrub(&mut value);
value
}

/// Debug wrapper over the request header map that prints only the key
/// headers, so the hot log line stays small and the sensitive headers never
/// reach the logs.
struct KeyRequestHeader<'a>(&'a HashMap<String, String>);

/// Debug implementation that filters the headers by the key header names.
impl fmt::Debug for KeyRequestHeader<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_map()
.entries(self.0.iter().filter(|(key, _)| is_key_request_header(key)))
.finish()
}
}

/// Returns whether the header is worth printing on the per-request log path,
/// matched case-insensitively without allocating.
fn is_key_request_header(key: &str) -> bool {
[
RANGE.as_str(),
USER_AGENT.as_str(),
HOST.as_str(),
"x-request-id",
]
.iter()
.any(|key_header| key.eq_ignore_ascii_case(key_header))
}

/// Debug-safe wrapper around [`DownloadPersistentTaskRequest`].
///
/// Use at log sites that would otherwise Debug-print the full
Expand Down Expand Up @@ -217,18 +182,92 @@ impl fmt::Debug for RedactedDownloadPersistentTaskRequest<'_> {
}
}

/// Debug-safe wrapper around [`http::Request`] received by the proxy.
///
/// Use at log sites that would otherwise Debug-print the full request
/// (e.g. `info!("proxy HTTP request: {:?}", request)`):
///
/// ```ignore
/// use crate::types::redacted::RedactedHTTPProxyRequest;
/// info!("proxy HTTP request: {:?}", RedactedHTTPProxyRequest(&request));
/// ```
///
/// Only the `method`, `uri`, `version` and the key headers are printed, so
/// the sensitive headers and the body never reach the logs.
pub struct RedactedHTTPProxyRequest<'a, B>(pub &'a Request<B>);

/// Debug implementation that prints the request line and the key headers
/// without the body, since it runs on the per-request log path.
impl<B> fmt::Debug for RedactedHTTPProxyRequest<'_, B> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Request")
.field("method", self.0.method())
.field("uri", self.0.uri())
.field("version", &self.0.version())
.field("headers", &KeyRequestHeader(self.0.headers()))
.finish_non_exhaustive()
}
}

/// Debug wrapper over the request header map that prints only the key
/// headers, so the hot log line stays small and the sensitive headers never
/// reach the logs.
struct KeyRequestHeader<'a, T>(&'a T);

/// Debug implementation that filters the headers by the key header names.
impl<'a, T, K, V> fmt::Debug for KeyRequestHeader<'a, T>
where
&'a T: IntoIterator<Item = (K, V)>,
K: AsRef<str> + fmt::Debug,
V: fmt::Debug,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_map()
.entries(
self.0
.into_iter()
.filter(|(key, _)| is_key_request_header(key.as_ref())),
)
.finish()
}
}

/// Returns whether the header is worth printing on the per-request log path,
/// matched case-insensitively without allocating.
fn is_key_request_header(key: &str) -> bool {
[
RANGE.as_str(),
USER_AGENT.as_str(),
HOST.as_str(),
"x-request-id",
]
.iter()
.any(|key_header| key.eq_ignore_ascii_case(key_header))
}

/// Clones the optional sub-message only when present and scrubs its secrets,
/// so the common absent case costs nothing.
fn scrubbed<T: Clone>(value: &Option<T>, scrub: fn(&mut Option<T>)) -> Option<T> {
let mut value = value.clone();
scrub(&mut value);
value
}

/// Mutates the given `ObjectStorage` by replacing any present secrets with [`REDACTED`].
fn scrub_object_storage(os: &mut Option<ObjectStorage>) {
if let Some(os) = os.as_mut() {
if os.access_key_secret.is_some() {
os.access_key_secret = Some(REDACTED.to_string());
}

if os.session_token.is_some() {
os.session_token = Some(REDACTED.to_string());
}

if os.security_token.is_some() {
os.security_token = Some(REDACTED.to_string());
}

if os.credential_path.is_some() {
os.credential_path = Some(REDACTED.to_string());
}
Expand Down Expand Up @@ -276,6 +315,7 @@ mod tests {
#![allow(clippy::type_complexity)]

use super::*;
use std::collections::HashMap;

const ACCESS_KEY_SECRET: &str = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";

Expand Down Expand Up @@ -584,4 +624,42 @@ mod tests {
Some(ACCESS_KEY_SECRET)
);
}

#[test]
fn redacted_http_proxy_request_debug_prints_the_request_line() {
let request = Request::builder()
.method("GET")
.uri("http://example.com/path")
.body(())
.unwrap();

let output = format!("{:?}", RedactedHTTPProxyRequest(&request));
assert!(output.contains("GET"));
assert!(output.contains("http://example.com/path"));
assert!(output.contains("HTTP/1.1"));
}

#[test]
fn redacted_http_proxy_request_debug_prints_only_key_request_headers() {
let test_cases = vec![
("range", "bytes=0-1023", true),
("user-agent", "dfget/2.3", true),
("host", "registry.example.com", true),
("x-request-id", "req-42", true),
("accept", "application/octet-stream", false),
("authorization", "Bearer SECRET_BEARER", false),
];

for (name, value, expected) in test_cases {
let request = Request::builder()
.uri("http://example.com/path")
.header(name, value)
.body(())
.unwrap();

let output = format!("{:?}", RedactedHTTPProxyRequest(&request));
assert_eq!(output.contains(name), expected);
assert_eq!(output.contains(value), expected);
}
}
}
30 changes: 17 additions & 13 deletions dragonfly-client/src/proxy/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ use dragonfly_client_util::{
http::{hashmap_to_headermap, headermap_to_hashmap},
shutdown,
tls::{generate_self_signed_certs_by_ca_cert, generate_simple_self_signed_certs, NoVerifier},
types::redacted::RedactedHTTPProxyRequest,
};
use futures::TryStreamExt;
use http_body_util::{combinators::BoxBody, BodyExt, Empty, StreamBody};
Expand Down Expand Up @@ -479,7 +480,7 @@ pub async fn http_handler(
if request.method() == Method::GET {
info!(
"proxy HTTP request via dfdaemon by rule config: {:?}",
request
RedactedHTTPProxyRequest(&request)
);

return proxy_via_dfdaemon(config.clone(), task, dynconfig, rule, request, remote_ip)
Expand All @@ -488,7 +489,7 @@ pub async fn http_handler(

debug!(
"proxy HTTP request bypassing dfdaemon for non-GET method: {:?}",
request
RedactedHTTPProxyRequest(&request)
);
}

Expand All @@ -497,7 +498,7 @@ pub async fn http_handler(
if header::get_use_p2p(request.headers()) {
info!(
"proxy HTTP request via dfdaemon by X-Dragonfly-Use-P2P header: {:?}",
request
RedactedHTTPProxyRequest(&request)
);

if request.method() == Method::GET {
Expand All @@ -514,22 +515,22 @@ pub async fn http_handler(

debug!(
"proxy HTTP request bypassing dfdaemon for non-GET method: {:?}",
request
RedactedHTTPProxyRequest(&request)
);
}

if request.uri().scheme().cloned() == Some(http::uri::Scheme::HTTPS) {
info!(
"proxy HTTPS request directly to remote server: {:?}",
request
RedactedHTTPProxyRequest(&request)
);

return proxy_via_https(request, https_client).await;
}

info!(
"proxy HTTP request directly to remote server: {:?}",
request
RedactedHTTPProxyRequest(&request)
);

return proxy_via_http(request, http_client).await;
Expand All @@ -549,7 +550,10 @@ pub async fn https_handler(
request_rate_limiter: Arc<RateLimiter>,
dynconfig: Arc<Dynconfig>,
) -> ClientResult<Response> {
debug!("handle HTTPS request: {:?}", request);
debug!(
"handle HTTPS request: {:?}",
RedactedHTTPProxyRequest(&request)
);

// Proxy the request directly to the remote server.
if let Some(host) = request.uri().host() {
Expand Down Expand Up @@ -763,7 +767,7 @@ pub async fn upgraded_handler(
if request.method() == Method::GET {
info!(
"proxy HTTPS request via dfdaemon by rule config: {:?}",
request,
RedactedHTTPProxyRequest(&request),
);

return proxy_via_dfdaemon(config.clone(), task, dynconfig, rule, request, remote_ip)
Expand All @@ -772,7 +776,7 @@ pub async fn upgraded_handler(

debug!(
"proxy HTTPS request bypassing dfdaemon for non-GET method: {:?}",
request,
RedactedHTTPProxyRequest(&request),
);
}

Expand All @@ -782,7 +786,7 @@ pub async fn upgraded_handler(
if request.method() == Method::GET {
info!(
"proxy HTTP request via dfdaemon by X-Dragonfly-Use-P2P header: {:?}",
request,
RedactedHTTPProxyRequest(&request),
);

return proxy_via_dfdaemon(
Expand All @@ -798,22 +802,22 @@ pub async fn upgraded_handler(

debug!(
"proxy HTTPS request bypassing dfdaemon for non-GET method: {:?}",
request,
RedactedHTTPProxyRequest(&request),
);
}

if request.uri().scheme().cloned() == Some(http::uri::Scheme::HTTPS) {
info!(
"proxy HTTPS request directly to remote server: {:?}",
request,
RedactedHTTPProxyRequest(&request),
);

return proxy_via_https(request, https_client).await;
}

info!(
"proxy HTTP request directly to remote server: {:?}",
request,
RedactedHTTPProxyRequest(&request),
);

return proxy_via_http(request, http_client).await;
Expand Down
Loading