From 757a0240daacfa4938dd8ad47f346bfb9bd0e77f Mon Sep 17 00:00:00 2001 From: Andrii Shestakov Date: Wed, 12 Nov 2025 14:04:08 +0100 Subject: [PATCH] Fix: shutdown supervisord when slurmd failing to (re)start --- internal/render/worker/configmap.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/render/worker/configmap.go b/internal/render/worker/configmap.go index 7a50d33d6..2504653d6 100644 --- a/internal/render/worker/configmap.go +++ b/internal/render/worker/configmap.go @@ -72,7 +72,7 @@ func generateDefaultSupervisordConfig() renderutils.ConfigFile { res.AddLine("command=/opt/bin/slurm/slurmd_entrypoint.sh") res.AddLine("autostart=true") res.AddLine("autorestart=true") - res.AddLine("startsecs=0") + res.AddLine("startsecs=10") res.AddLine("stopasgroup=true ; Send SIGTERM to all child processes of supervisord") res.AddLine("killasgroup=true ; Send SIGKILL to all child processes of supervisord") res.AddLine("stopsignal=SIGTERM ; Signal to send to the program to stop it") @@ -93,6 +93,10 @@ func generateDefaultSupervisordConfig() renderutils.ConfigFile { res.AddLine("killasgroup=true ; Send SIGKILL to all child processes of supervisord") res.AddLine("stopsignal=SIGTERM ; Signal to send to the program to stop it") res.AddLine("stopwaitsecs=10 ; Wait for the process to stop before sending a SIGKILL") + res.AddLine("") + res.AddLine("[eventlistener:quit_on_failure]") + res.AddLine("events=PROCESS_STATE_FATAL") + res.AddLine("command=sh -c 'echo \"READY\"; while read -r line; do echo \"$line\"; supervisorctl shutdown; done'") return res }