diff --git a/docs/security/security-model.md b/docs/security/security-model.md index ca5647a71..14c8cd5a3 100644 --- a/docs/security/security-model.md +++ b/docs/security/security-model.md @@ -387,6 +387,14 @@ This argument has a limit, and it is worth stating because it is what keeps the Production verifiers should reject deployments that use these development settings. Operators should treat them the same way they treat debug-mode TEE quotes: useful for testing, invalid for production trust. +### The auth API channel is not attested, and the topology decides whether that matters + +`auth_api.type = "webhook"` makes an external HTTP service the authorization decision: `bootAuth/app` returns `isAllowed` and `gatewayAppId`, and the KMS releases app keys on that answer. The KMS uses a bare HTTP client for it -- no CA pinning, no shared secret, and no signature over the response -- so the channel carries exactly the trust the network under it carries. + +That is sound in the topology the shipped `dstack-app/docker-compose.yaml` uses, where the backend runs in the same compose network inside the CVM: one trust domain, no network in between. It is not sound in `dstack-app/compose-simple.yaml`, whose own header documents the backend as running outside the CVM on operator infrastructure -- there, anything on the path between the KMS and the backend can turn a deny into an allow for a `BootInfo` the app owner never authorized. + +The KMS cannot tell the two apart with certainty from a URL, so it warns at startup rather than refusing: loopback and a single-label host (a compose service name, which cannot be a public DNS name) are taken as in-CVM, and anything else over `http://` is warned about. `deploy-simple.sh` -- which knows it is the external topology -- refuses anything but `https://`. TLS here authenticates the backend's name, not an attestation; a response signed by a key the KMS pins would be the stronger fix and would need all three backends to implement it. + ### KMS mTLS is route-enforced for sensitive operations The KMS Rocket TLS listener permits connections without a client certificate because some bootstrap and public metadata endpoints must be reachable before a client has an RA-TLS certificate. A certificate that is presented must carry an attestation, but the issuer that signed it is not checked and is not the authorization boundary for key material. diff --git a/dstack/kms/auth-simple/README.md b/dstack/kms/auth-simple/README.md index 6857122d1..e4e25b32c 100644 --- a/dstack/kms/auth-simple/README.md +++ b/dstack/kms/auth-simple/README.md @@ -232,6 +232,9 @@ Configure KMS to use webhook auth pointing to this server: type = "webhook" [core.auth_api.webhook] +# Plain HTTP is only appropriate when auth-simple runs inside the CVM, as here. +# Serve it over https:// whenever it does not - the KMS releases app keys on +# this backend's answer and nothing authenticates that answer on the wire. url = "http://localhost:3000" ``` diff --git a/dstack/kms/dstack-app/compose-simple.yaml b/dstack/kms/dstack-app/compose-simple.yaml index 711b06cad..ce2bdeae3 100644 --- a/dstack/kms/dstack-app/compose-simple.yaml +++ b/dstack/kms/dstack-app/compose-simple.yaml @@ -4,6 +4,12 @@ # KMS deployment with external auth-simple webhook # auth-simple runs outside the CVM (operator infrastructure) +# +# AUTH_WEBHOOK_URL must be https://. The KMS releases app keys on whatever this +# backend answers, and because the backend is outside the CVM that answer +# crosses a network the KMS does not control. There is no CA pinning, no shared +# secret and no signature over the response, so over plain HTTP anything on the +# path can turn a deny into an allow. The KMS warns at startup if it is http://. services: kms: diff --git a/dstack/kms/dstack-app/deploy-simple.sh b/dstack/kms/dstack-app/deploy-simple.sh index c24b37715..d212e9198 100755 --- a/dstack/kms/dstack-app/deploy-simple.sh +++ b/dstack/kms/dstack-app/deploy-simple.sh @@ -26,8 +26,10 @@ else # Required: The URL of the dstack-vmm RPC service # VMM_RPC=unix:../../../build/vmm.sock -# Required: External auth-simple webhook URL (running on operator infrastructure) -# AUTH_WEBHOOK_URL=http://your-auth-server:3001 +# Required: External auth-simple webhook URL (running on operator infrastructure). +# Must be https://: the KMS releases app keys on this backend's answer, and +# nothing authenticates that answer on the wire. +# AUTH_WEBHOOK_URL=https://your-auth-server:3001 # Required: The address of the KMS service listening on Host machine # KMS_RPC_ADDR=0.0.0.0:9201 @@ -71,6 +73,22 @@ for var in "${required_env_vars[@]}"; do fi done +# auth-simple runs outside the CVM in this topology, so its answer - which is +# the whole authorization decision - crosses a network the KMS does not +# control. Nothing pins, signs or authenticates it, so refuse plain HTTP here +# rather than shipping a deployment that trusts the path. +case "$AUTH_WEBHOOK_URL" in + https://*) ;; + http://localhost* | http://127.0.0.1*) + echo "Error: AUTH_WEBHOOK_URL points at loopback, but auth-simple runs outside the CVM in this topology." + exit 1 + ;; + *) + echo "Error: AUTH_WEBHOOK_URL must be https://; the KMS releases app keys on this backend's answer and nothing authenticates it over plain HTTP." + exit 1 + ;; +esac + CLI=(../../vmm/src/vmm-cli.py --url "$VMM_RPC") COMPOSE_TMP=$(mktemp) diff --git a/dstack/kms/src/main_service.rs b/dstack/kms/src/main_service.rs index a0b91cf79..220d59358 100644 --- a/dstack/kms/src/main_service.rs +++ b/dstack/kms/src/main_service.rs @@ -158,6 +158,7 @@ impl KmsState { "self-authorization is disabled; trusted RPCs will not be gated by KMS self-attestation - do not use in production TEE deployments" ); } + upgrade_authority::warn_on_unauthenticated_auth_api(&config.auth_api); Ok(Self { inner: Arc::new(KmsStateInner { config, diff --git a/dstack/kms/src/main_service/upgrade_authority.rs b/dstack/kms/src/main_service/upgrade_authority.rs index 135f49de4..c25c04b4f 100644 --- a/dstack/kms/src/main_service/upgrade_authority.rs +++ b/dstack/kms/src/main_service/upgrade_authority.rs @@ -12,6 +12,7 @@ use http_client::prpc::PrpcClient; use ra_tls::attestation::{AttestationVerifier, VerifiedAttestation, VersionedAttestation}; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; +use tracing::warn; /// The KMS `bootAuth` payload. This is the verifier's `PolicyBootInfo` — the one /// canonical struct shared by the producer (KMS) and the policy input the @@ -184,6 +185,62 @@ impl AuthApi { } } +/// Whether the auth API is reached over a channel that nothing authenticates. +/// +/// The auth API *is* the authorization decision: `is_app_allowed` returns +/// `isAllowed` and `gatewayAppId`, and the KMS releases app keys on that +/// answer. `send_request` uses a bare `reqwest::Client`, so a plain-HTTP +/// webhook URL means anything on the path between the KMS and the backend can +/// turn a deny into an allow for a `BootInfo` the app owner never authorized. +/// There is no CA pinning, no shared secret and no signature over the response +/// to fall back on, and TLS alone would only authenticate the backend's name - +/// but that is still the difference between "an on-path attacker" and "whoever +/// holds the backend's certificate". +/// +/// Two host shapes are exempt because they cannot reach off the machine, and +/// they are exactly the two in-CVM topologies dstack ships. Loopback is +/// `auth-simple`'s documented `http://localhost:3000`. A single-label host is +/// the shipped default `http://auth-api:8000` - a compose service name, which +/// resolves through the container's own resolver and is not a public DNS name. +/// In both the backend runs inside the CVM, which is one trust domain. +/// +/// Anything else is a name that can resolve to another machine, which is the +/// `dstack-app/compose-simple.yaml` topology - its own header says the backend +/// runs outside the CVM. This only warns rather than refusing, because the +/// exemption above is a heuristic and the shipped default depends on it; +/// `deploy-simple.sh`, which knows it is the external topology, refuses. +fn webhook_channel_is_unauthenticated(url: &str) -> bool { + let Some(rest) = url.strip_prefix("http://") else { + return false; + }; + let authority = rest.split(['/', '?', '#']).next().unwrap_or_default(); + let host = match authority.rsplit_once(':') { + Some((host, port)) if port.chars().all(|c| c.is_ascii_digit()) => host, + _ => authority, + }; + let host = host.trim_start_matches('[').trim_end_matches(']'); + if matches!(host, "localhost" | "127.0.0.1" | "::1") { + return false; + } + host.contains('.') +} + +/// Warn once at startup when the authorization decision travels over a channel +/// nothing authenticates. See [`webhook_channel_is_unauthenticated`]. +pub(crate) fn warn_on_unauthenticated_auth_api(auth_api: &AuthApi) { + let AuthApi::Webhook { webhook } = auth_api else { + return; + }; + if webhook_channel_is_unauthenticated(&webhook.url) { + warn!( + "auth api {} is reached over plain HTTP; the KMS releases app keys on this \ + backend's answer, and nothing authenticates it - use https:// whenever the \ + backend does not run inside this CVM", + webhook.url + ); + } +} + fn url_join(url: &str, path: &str) -> String { let mut url = url.to_string(); if !url.ends_with('/') { @@ -363,6 +420,38 @@ mod tests { } } + /// The `os_image_hash` substitution on the onboarding path is for source + /// KMS instances whose certificate predates `vm_config`, and it costs + /// `allowedOsImages` its say over the source. Keep it scoped to exactly + /// that case: a source that does present a config and still reports no + /// image keeps an empty hash and is denied on-chain, because + /// `allowedOsImages[bytes32(0)]` is false. + /// The authorization decision must not travel over a channel nothing + /// authenticates once the backend leaves the CVM. Loopback and the + /// compose-network name stay quiet because those are the in-CVM + /// topologies the shipped compose files use. + #[test] + fn a_plain_http_auth_api_outside_the_cvm_is_flagged() { + for quiet in [ + "https://auth.example.com", + "https://auth.example.com:3001/", + "http://localhost:3000", + "http://127.0.0.1:3000/bootAuth", + "http://[::1]:3000", + // the shipped default: a compose service name, in-CVM + "http://auth-api:8000", + ] { + assert!(!webhook_channel_is_unauthenticated(quiet), "{quiet}"); + } + for flagged in [ + "http://auth.example.com", + "http://auth.example.com:3001/", + "http://10.0.0.5:3001", + ] { + assert!(webhook_channel_is_unauthenticated(flagged), "{flagged}"); + } + } + #[rocket::async_test] async fn repeated_authorization_is_never_served_from_a_decision_cache() { let (url, server) = serve(vec![