certbot: Support config for max DNS wait time

certbot/cli/src/main.rs

@@ -74,6 +74,8 @@ struct Config {
     renew_days_before: u64,
     /// Renew timeout in seconds
     renew_timeout: u64,
+    /// Maximum time to wait for DNS propagation in seconds
+    max_dns_wait: u64,
     /// Command to run after renewal
     #[serde(default)]
     renewed_hook: Option<String>,
@@ -91,6 +93,7 @@ impl Default for Config {
             renew_interval: 3600,
             renew_days_before: 10,
             renew_timeout: 120,
+            max_dns_wait: 300,
             renewed_hook: None,
         }
     }
@@ -125,6 +128,7 @@ fn load_config(config: &PathBuf) -> Result<CertBotConfig> {
     let renew_interval = Duration::from_secs(config.renew_interval);
     let renew_expires_in = Duration::from_secs(config.renew_days_before * 24 * 60 * 60);
     let renew_timeout = Duration::from_secs(config.renew_timeout);
+    let max_dns_wait = Duration::from_secs(config.max_dns_wait);
     let bot_config = CertBotConfig::builder()
         .acme_url(config.acme_url)
         .cert_dir(workdir.backup_dir())
@@ -137,6 +141,7 @@ fn load_config(config: &PathBuf) -> Result<CertBotConfig> {
         .renew_interval(renew_interval)
         .renew_timeout(renew_timeout)
         .renew_expires_in(renew_expires_in)
+        .max_dns_wait(max_dns_wait)
         .credentials_file(workdir.account_credentials_path())
         .auto_set_caa(config.auto_set_caa)
         .maybe_renewed_hook(config.renewed_hook)

certbot/src/acme_client.rs

@@ -27,6 +27,7 @@ pub struct AcmeClient {
     account: Account,
     credentials: Credentials,
     dns01_client: Dns01Client,
+    max_dns_wait: Duration,
 }
 
 #[derive(Debug, Clone)]
@@ -53,19 +54,28 @@ pub(crate) fn acme_matches(encoded_credentials: &str, acme_url: &str) -> bool {
 }
 
 impl AcmeClient {
-    pub async fn load(dns01_client: Dns01Client, encoded_credentials: &str) -> Result<Self> {
+    pub async fn load(
+        dns01_client: Dns01Client,
+        encoded_credentials: &str,
+        max_dns_wait: Duration,
+    ) -> Result<Self> {
         let credentials: Credentials = serde_json::from_str(encoded_credentials)?;
         let account = Account::from_credentials(credentials.credentials).await?;
         let credentials: Credentials = serde_json::from_str(encoded_credentials)?;
         Ok(Self {
             account,
             dns01_client,
             credentials,
+            max_dns_wait,
         })
     }
 
     /// Create a new account.
-    pub async fn new_account(acme_url: &str, dns01_client: Dns01Client) -> Result<Self> {
+    pub async fn new_account(
+        acme_url: &str,
+        dns01_client: Dns01Client,
+        max_dns_wait: Duration,
+    ) -> Result<Self> {
         let (account, credentials) = Account::create(
             &NewAccount {
                 contact: &[],
@@ -86,6 +96,7 @@ impl AcmeClient {
             account,
             dns01_client,
             credentials,
+            max_dns_wait,
         })
     }
 
@@ -335,18 +346,31 @@ impl AcmeClient {
 
     /// Self check the TXT records for the given challenges.
     async fn check_dns(&self, challenges: &[Challenge]) -> Result<()> {
+        use tracing::warn;
+
         let mut delay = Duration::from_millis(250);
         let mut tries = 1u8;
 
         let mut unsettled_challenges = challenges.to_vec();
 
         debug!("Unsettled challenges: {unsettled_challenges:#?}");
 
+        let start_time = std::time::Instant::now();
+
         'outer: loop {
             use hickory_resolver::AsyncResolver;
 
             sleep(delay).await;
 
+            let elapsed = start_time.elapsed();
+            if elapsed >= self.max_dns_wait {
+                warn!(
+                    "DNS propagation timeout after {elapsed:?}, max wait time is {max:?}. proceeding anyway as ACME server may have different DNS view",
+                    max = self.max_dns_wait
+                );
+                break;
+            }
+
             let dns_resolver =
                 AsyncResolver::tokio_from_system_conf().context("failed to create dns resolver")?;
 
@@ -374,6 +398,8 @@ impl AcmeClient {
                     debug!(
                         tries,
                         domain = &challenge.acme_domain,
+                        elapsed = ?elapsed,
+                        max_wait = ?self.max_dns_wait,
                         "challenge not found, waiting for {delay:?}"
                     );
                     unsettled_challenges.push(challenge);

certbot/src/acme_client/tests.rs

@@ -13,7 +13,7 @@ async fn new_acme_client() -> Result<AcmeClient> {
     );
     let credentials =
         std::env::var("LETSENCRYPT_CREDENTIAL").expect("LETSENCRYPT_CREDENTIAL not set");
-    AcmeClient::load(dns01_client, &credentials).await
+    AcmeClient::load(dns01_client, &credentials, Duration::from_secs(300)).await
 }
 
 #[tokio::test]

certbot/src/bot.rs

@@ -37,6 +37,7 @@ pub struct CertBotConfig {
     renew_timeout: Duration,
     renew_expires_in: Duration,
     renewed_hook: Option<String>,
+    max_dns_wait: Duration,
 }
 
 impl CertBotConfig {
@@ -55,7 +56,7 @@ async fn create_new_account(
     dns01_client: Dns01Client,
 ) -> Result<AcmeClient> {
     info!("creating new ACME account");
-    let client = AcmeClient::new_account(&config.acme_url, dns01_client)
+    let client = AcmeClient::new_account(&config.acme_url, dns01_client, config.max_dns_wait)
         .await
         .context("failed to create new account")?;
     let credentials = client
@@ -82,7 +83,7 @@ impl CertBot {
         let acme_client = match fs::read_to_string(&config.credentials_file) {
             Ok(credentials) => {
                 if acme_matches(&credentials, &config.acme_url) {
-                    AcmeClient::load(dns01_client, &credentials).await?
+                    AcmeClient::load(dns01_client, &credentials, config.max_dns_wait).await?
                 } else {
                     create_new_account(&config, dns01_client).await?
                 }

certbot/src/bot/tests.rs

@@ -25,6 +25,7 @@ async fn new_certbot() -> Result<CertBot> {
         .renew_interval(Duration::from_secs(30))
         .renew_timeout(Duration::from_secs(120))
         .renew_expires_in(Duration::from_secs(7772187))
+        .max_dns_wait(Duration::from_secs(300))
         .auto_set_caa(false)
         .build();
     config.build_bot().await

gateway/dstack-app/builder/entrypoint.sh

@@ -119,6 +119,7 @@ domain = "*.$SRV_DOMAIN"
 renew_interval = "1h"
 renew_before_expiration = "10d"
 renew_timeout = "5m"
+max_dns_wait = "${CERTBOT_MAX_DNS_WAIT:-5m}"
 
 [core.wg]
 public_key = "$PUBLIC_KEY"

gateway/dstack-app/deploy-to-vmm.sh

@@ -84,6 +84,8 @@ GATEWAY_SERVING_ADDR=0.0.0.0:9204
 GUEST_AGENT_ADDR=127.0.0.1:9206
 WG_ADDR=0.0.0.0:9202
 
+CERTBOT_MAX_DNS_WAIT=5m
+
 # The token used to launch the App
 APP_LAUNCH_TOKEN=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
 
@@ -143,6 +145,7 @@ BOOTNODE_URL=$BOOTNODE_URL
 SUBNET_INDEX=$SUBNET_INDEX
 APP_LAUNCH_TOKEN=$APP_LAUNCH_TOKEN
 RPC_DOMAIN=$RPC_DOMAIN
+CERTBOT_MAX_DNS_WAIT=$CERTBOT_MAX_DNS_WAIT
 EOF
 
 if [ -n "$APP_COMPOSE_FILE" ]; then

gateway/gateway.toml

@@ -38,6 +38,7 @@ domain = "*.example.com"
 renew_interval = "1h"
 renew_before_expiration = "10d"
 renew_timeout = "120s"
+max_dns_wait = "5m"
 
 [core.wg]
 public_key = ""

gateway/src/config.rs

@@ -208,6 +208,9 @@ pub struct CertbotConfig {
     /// Renew timeout
     #[serde(with = "serde_duration")]
     pub renew_timeout: Duration,
+    /// Maximum time to wait for DNS propagation
+    #[serde(with = "serde_duration")]
+    pub max_dns_wait: Duration,
 }
 
 impl CertbotConfig {
@@ -227,6 +230,7 @@ impl CertbotConfig {
             .renew_timeout(self.renew_timeout)
             .renew_expires_in(self.renew_before_expiration)
             .auto_set_caa(self.auto_set_caa)
+            .max_dns_wait(self.max_dns_wait)
             .build()
     }