fix(kms): move KMS key handover to the admin listener

docs/deployment.md

@@ -492,7 +492,7 @@ port = 9203                  # HTTP port for onboard UI
 ```bash
 curl -X POST http://<new-kms>:9203/prpc/Onboard.Onboard?json \
   -H "Content-Type: application/json" \
-  -d '{"source_url": "https://<existing-kms>:9201/prpc", "domain": "kms2.example.com"}'
+  -d '{"source_url": "https://<existing-kms-admin>:9202/prpc", "source_token": "<optional-admin-token>", "domain": "kms2.example.com"}'
 ```
 
 **Finish and restart:**
@@ -509,7 +509,9 @@ curl http://<new-kms>:9203/finish
 >
 > If you skip this, `Onboard.Onboard` or later trusted RPCs will fail with KMS authorization errors.
 
-> **Admin authentication.** Onboarding itself is gated by attestation and the authorization backend above — not by a token. Separately, the KMS *admin* RPCs (for example `ClearImageCache`) are served on a dedicated `[core.admin]` listener behind the shared HTTP authenticator, just like the VMM and gateway: set `[core.admin] enabled = true` with an `auth_token` (or the `DSTACK_KMS_ADMIN_TOKEN` / `ADMIN_API_TOKEN` env vars), and clients send `Authorization: Bearer <token>` or `X-Admin-Token`. Enabled with neither `auth_token` nor `htpasswd_file` (and `insecure_no_auth = false`) fails closed — the KMS refuses to start. See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication).
+> **Admin authentication.** The KMS *admin* RPCs (for example `ClearImageCache`) are served on a dedicated `[core.admin]` listener behind the shared HTTP authenticator, just like the VMM and gateway: set `[core.admin] enabled = true` with an `auth_token` (or the `DSTACK_KMS_ADMIN_TOKEN` / `ADMIN_API_TOKEN` env vars), and clients send `Authorization: Bearer <token>` or `X-Admin-Token`. Enabled with neither `auth_token` nor `htpasswd_file` (and `insecure_no_auth = false`) fails closed — the KMS refuses to start. See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication).
+>
+> **Key handover.** It is served as `Admin.GetKmsKey` on the `[core.admin]` listener, which needs `[core.admin.tls]` so it can verify the onboarding KMS's RA-TLS certificate. Pass the source admin URL and its token (`source_token`) in the onboarding request. `core.onboard.public_key_handover = true` keeps the legacy public `KMS.GetKmsKey`.
 
 ---
 

docs/security/security-model.md

@@ -417,6 +417,10 @@ App key release and KMS key handover require verified caller attestation from th
 
 The unauthenticated or non-client-certificate surface includes bootstrap and temp-CA bootstrap material retrieval, env-encryption public-key retrieval, metadata, health, and metrics behavior documented for operators. `GetTempCaCert` returns temp CA private material and remains in use by guests and by KMS-to-KMS onboarding, which mint their client certificates from that CA; operators must treat it as bootstrap-sensitive rather than harmless public metadata.
 
+KMS-to-KMS root-key handover is served by `Admin.GetKmsKey` on the authenticated
+admin listener, which additionally requires a verified RA-TLS client certificate.
+The legacy public `KMS.GetKmsKey` is controlled by `core.onboard.public_key_handover`.
+
 ## Limitations
 
 ### Attestation proves identity, not correctness

dstack/kms/README.md

@@ -33,7 +33,7 @@ CVMs running in dstack support three boot modes:
 ## Admin API authentication
 
 The KMS public RPCs authenticate callers by RA-TLS attestation. Operator-facing
-admin RPCs (currently `ClearImageCache`) are instead served on a **separate admin
+admin RPCs are instead served on a **separate admin
 listener** (`[core.admin]` in `kms.toml`) behind the same shared HTTP
 authenticator used by the VMM and gateway, configured identically to the gateway
 admin API, so the credential travels in the `Authorization: Bearer <token>` or
@@ -48,6 +48,10 @@ port = 8001
 auth_token = "<token>"
 # htpasswd_file = "/etc/kms/admin.htpasswd"   # bcrypt (htpasswd -B), optional
 insecure_no_auth = false
+
+[core.admin.tls]
+key = "/etc/kms/certs/rpc.key"
+certs = "/etc/kms/certs/rpc.crt"
 ```
 
 The token can also be supplied via the `DSTACK_KMS_ADMIN_TOKEN` or
@@ -65,6 +69,12 @@ curl -X POST "http://127.0.0.1:8001/prpc/Admin.ClearImageCache?json" \
 This admin authentication is separate from the on-chain / webhook authorization
 below, which decides whether a CVM may boot and receive keys.
 
+KMS-to-KMS onboarding calls `Admin.GetKmsKey` with a self-issued RA-TLS client
+certificate plus the source's admin bearer token; other admin RPCs need no client
+certificate. `core.onboard.public_key_handover = true` (the default) keeps
+`KMS.GetKmsKey` on the public listener for older clients.
+
+
 ## KMS Implementation
 
 ### Components

dstack/kms/kms.toml

@@ -101,6 +101,12 @@ htpasswd_file = ""
 # interface that is reachable from the network.
 insecure_no_auth = false
 
+# Optional TLS for the admin listener. When configured, clients may present an
+# RA-TLS certificate. GetKmsKey requires it; other admin RPCs do not.
+# [core.admin.tls]
+# key = "/etc/kms/certs/rpc.key"
+# certs = "/etc/kms/certs/rpc.crt"
+
 [core.metrics]
 # Expose unauthenticated Prometheus metrics on /metrics.
 # Disable this if the KMS RPC listener is not protected by network policy
@@ -121,5 +127,8 @@ gateway_app_id = "any"
 [core.onboard]
 enabled = true
 auto_bootstrap_domain = ""
+# Keep KMS.GetKmsKey on the public RPC listener for older onboarding clients.
+# New deployments should point onboarding at the admin listener instead.
+public_key_handover = true
 address = "0.0.0.0"
 port = 8000

dstack/kms/rpc/proto/kms_rpc.proto

@@ -134,6 +134,8 @@ service KMS {
 service Admin {
   // Clear the image cache
   rpc ClearImageCache(ClearImageCacheRequest) returns (google.protobuf.Empty);
+  // Hand root keys to an attested KMS during onboarding.
+  rpc GetKmsKey(GetKmsKeyRequest) returns (KmsKeyResponse);
 }
 
 message ClearImageCacheRequest {
@@ -154,6 +156,8 @@ message BootstrapResponse {
 message OnboardRequest {
   string source_url = 1;
   string domain = 2;
+  // Optional bearer token for the source KMS admin listener.
+  string source_token = 3;
 }
 
 message OnboardResponse {

dstack/kms/src/admin_service.rs

@@ -10,21 +10,26 @@
 use anyhow::Result;
 use dstack_kms_rpc::{
     admin_server::{AdminRpc, AdminServer},
-    ClearImageCacheRequest,
+    ClearImageCacheRequest, GetKmsKeyRequest, KmsKeyResponse,
 };
 use ra_rpc::{CallContext, RpcCall};
 
-use crate::main_service::KmsState;
+use crate::main_service::{KmsState, RpcHandler};
 
 pub struct AdminRpcHandler {
     state: KmsState,
+    rpc: RpcHandler,
 }
 
 impl AdminRpc for AdminRpcHandler {
     async fn clear_image_cache(self, request: ClearImageCacheRequest) -> Result<()> {
         self.state
             .clear_image_cache(&request.image_hash, &request.config_hash)
     }
+
+    async fn get_kms_key(self, request: GetKmsKeyRequest) -> Result<KmsKeyResponse> {
+        self.rpc.handover_keys(request).await
+    }
 }
 
 impl RpcCall<KmsState> for AdminRpcHandler {
@@ -33,6 +38,7 @@ impl RpcCall<KmsState> for AdminRpcHandler {
     fn construct(context: CallContext<'_, KmsState>) -> Result<Self> {
         Ok(AdminRpcHandler {
             state: context.state.clone(),
+            rpc: RpcHandler::from_context(context),
         })
     }
 }

dstack/kms/src/config.rs

@@ -214,6 +214,9 @@ pub(crate) struct Dev {
 pub(crate) struct OnboardConfig {
     pub enabled: bool,
     pub auto_bootstrap_domain: String,
+    /// Keep GetKmsKey available on the public RPC listener for compatibility.
+    #[serde(default = "default_true")]
+    pub public_key_handover: bool,
 }
 
 #[cfg(test)]

dstack/kms/src/main.rs

@@ -178,6 +178,7 @@ async fn main() -> Result<()> {
     };
     let state = main_service::KmsState::new(config).context("Failed to initialize KMS state")?;
     let quote_verifier = QuoteVerifier::new(state.attestation_verifier());
+    let admin_quote_verifier = QuoteVerifier::new(state.attestation_verifier());
     let figment = figment
         .clone()
         .merge(Serialized::defaults(figment.find_value("rpc")?));
@@ -223,12 +224,14 @@ async fn main() -> Result<()> {
             }
             let admin_srv = rocket::custom(admin_figment)
                 .attach(admin_fairing)
+                .attach(RaTlsClientAuth::fairing())
                 .mount("/", admin_auth::routes())
                 .mount(
                     "/prpc",
                     ra_rpc::prpc_routes!(KmsState, admin_service::AdminRpcHandler, trim: "Admin."),
                 )
                 .manage(state)
+                .manage(admin_quote_verifier)
                 .launch();
             tokio::try_join!(
                 async { main_srv.await.map_err(|err| anyhow!(err.to_string())) },

dstack/kms/src/main_service.rs

@@ -214,6 +214,30 @@ pub struct RpcHandler {
     attestation: Option<VerifiedAttestation>,
 }
 
+impl RpcHandler {
+    pub(crate) fn from_context(context: CallContext<'_, KmsState>) -> Self {
+        Self {
+            state: context.state.clone(),
+            attestation: context.attestation,
+        }
+    }
+
+    pub(crate) async fn handover_keys(self, request: GetKmsKeyRequest) -> Result<KmsKeyResponse> {
+        self.ensure_self_allowed()
+            .await
+            .context("KMS self authorization failed")?;
+        let info = self.ensure_kms_allowed(&request.vm_config).await?;
+        ensure_key_release_allowed(&info, (&self.state.config).into())?;
+        Ok(KmsKeyResponse {
+            temp_ca_key: self.state.inner.temp_ca_key.clone(),
+            keys: vec![KmsKeys {
+                ca_key: self.state.inner.root_ca.key.serialize_pem(),
+                k256_key: self.state.inner.k256_key.to_bytes().to_vec(),
+            }],
+        })
+    }
+}
+
 struct BootConfig {
     boot_info: BootInfo,
     gateway_app_id: String,
@@ -534,18 +558,10 @@ impl KmsRpc for RpcHandler {
     }
 
     async fn get_kms_key(self, request: GetKmsKeyRequest) -> Result<KmsKeyResponse> {
-        self.ensure_self_allowed()
-            .await
-            .context("KMS self authorization failed")?;
-        let info = self.ensure_kms_allowed(&request.vm_config).await?;
-        ensure_key_release_allowed(&info, (&self.state.config).into())?;
-        Ok(KmsKeyResponse {
-            temp_ca_key: self.state.inner.temp_ca_key.clone(),
-            keys: vec![KmsKeys {
-                ca_key: self.state.inner.root_ca.key.serialize_pem(),
-                k256_key: self.state.inner.k256_key.to_bytes().to_vec(),
-            }],
-        })
+        if !self.state.config.onboard.public_key_handover {
+            bail!("public KMS key handover is disabled; use the admin listener");
+        }
+        self.handover_keys(request).await
     }
 
     /// Serve the temp CA certificate and key.
@@ -630,10 +646,7 @@ impl RpcCall<KmsState> for RpcHandler {
     type PrpcService = KmsServer<Self>;
 
     fn construct(context: CallContext<'_, KmsState>) -> Result<Self> {
-        Ok(RpcHandler {
-            state: context.state.clone(),
-            attestation: context.attestation,
-        })
+        Ok(RpcHandler::from_context(context))
     }
 }
 

dstack/kms/src/onboard_service.rs

@@ -152,6 +152,7 @@ impl OnboardRpc for OnboardHandler {
         let keys = Keys::onboard(
             cfg,
             &source_url,
+            &request.source_token,
             &request.domain,
             self.state.attestation_verifier.clone(),
         )
@@ -599,6 +600,7 @@ impl Keys {
     async fn onboard(
         cfg: &KmsConfig,
         other_kms_url: &str,
+        source_token: &str,
         domain: &str,
         attestation_verifier: Arc<AttestationVerifier>,
     ) -> Result<Self> {
@@ -621,6 +623,7 @@ impl Keys {
                 Ok(())
             }))
             .attestation_verifier(attestation_verifier.clone())
+            .maybe_bearer_token((!source_token.is_empty()).then(|| source_token.to_string()))
             .build()
             .into_client()?;
         let mut kms_client = KmsClient::new(client);

dstack/kms/src/www/onboard.html

@@ -61,6 +61,7 @@
         }
 
         input[type="text"],
+        input[type="password"],
         input[type="url"] {
             width: 100%;
             padding: 8px;
@@ -237,6 +238,11 @@ <h2>Onboard from an Existing KMS Instance</h2>
                                 <input type="url" id="source_url" v-model="sourceUrl" required
                                     placeholder="RPC URL of the existing KMS">
                             </div>
+                            <div class="form-group">
+                                <label for="source_token">Source admin token (optional):</label>
+                                <input type="password" id="source_token" v-model="sourceToken"
+                                    autocomplete="off">
+                            </div>
                             <div class="form-group">
                                 <label for="domain">Domain Name:</label>
                                 <input type="text" id="domain" v-model="onboardDomain" required
@@ -283,6 +289,7 @@ <h2>Onboard from an Existing KMS Instance</h2>
                     selectedOption: null,
                     bootstrapDomain: '',
                     sourceUrl: '',
+                    sourceToken: '',
                     onboardDomain: '',
                     error: '',
                     success: '',
@@ -337,6 +344,7 @@ <h2>Onboard from an Existing KMS Instance</h2>
                     try {
                         const data = await rpcCall('Onboard', {
                             source_url: this.sourceUrl + "/prpc",
+                            source_token: this.sourceToken,
                             domain: this.onboardDomain
                         });
 

dstack/ra-rpc/src/client.rs

@@ -39,6 +39,7 @@ pub struct RaClientConfig {
     tls_client_cert: Option<String>,
     tls_client_key: Option<String>,
     tls_ca_cert: Option<String>,
+    bearer_token: Option<String>,
     #[builder(default = true)]
     tls_built_in_root_certs: bool,
     attestation_verifier: Option<Arc<AttestationVerifier>>,
@@ -88,6 +89,7 @@ impl RaClientConfig {
             client,
             cert_validator: self.cert_validator,
             verify_server_attestation: self.verify_server_attestation,
+            bearer_token: self.bearer_token,
         })
     }
 }
@@ -98,6 +100,7 @@ pub struct RaClient {
     client: Client,
     cert_validator: Option<CertValidator>,
     verify_server_attestation: bool,
+    bearer_token: Option<String>,
 }
 
 impl RaClient {
@@ -209,13 +212,11 @@ impl RequestClient for RaClient {
     {
         let body = serde_json::to_vec(&body).context("Failed to serialize body")?;
         let url = format!("{}/{}?json", self.remote_uri, path);
-        let response = self
-            .client
-            .post(url)
-            .body(body)
-            .send()
-            .await
-            .context("Failed to send request")?;
+        let mut request = self.client.post(url).body(body);
+        if let Some(token) = &self.bearer_token {
+            request = request.bearer_auth(token);
+        }
+        let response = request.send().await.context("Failed to send request")?;
 
         // Name the direction explicitly: this validates the *server's* attestation,
         // not the client's own quote. Without it the error chain reads as if the