From e2b1547d57e94d0330f98cc17d55dd1823b9f1d8 Mon Sep 17 00:00:00 2001 From: Dmitry Starov Date: Fri, 17 Jul 2026 16:39:48 +0200 Subject: [PATCH 1/5] [ADD] Use `jobacct_gather/none` as default `JobAcctGatherType` Reduces REQUEST_SUBMIT_BATCH_JOB RPC time. --- api/v1/slurmcluster_types.go | 2 +- config/crd/bases/slurm.nebius.ai_slurmclusters.yaml | 2 +- helm/slurm-cluster/tests/README.md | 2 +- helm/slurm-cluster/tests/default-values_test.yaml | 2 +- helm/slurm-cluster/values.yaml | 2 +- helm/soperator-crds/templates/slurmcluster-crd.yaml | 2 +- helm/soperator/crds/slurmcluster-crd.yaml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/v1/slurmcluster_types.go b/api/v1/slurmcluster_types.go index 8d87c03dc..4ecd29c23 100644 --- a/api/v1/slurmcluster_types.go +++ b/api/v1/slurmcluster_types.go @@ -884,7 +884,7 @@ type AccountingSlurmConf struct { AcctGatherProfileType *string `json:"acctGatherProfileType,omitempty"` // +kubebuilder:validation:Optional // +kubebuilder:validation:Enum="jobacct_gather/linux";"jobacct_gather/cgroup";"jobacct_gather/none" - // +kubebuilder:default="jobacct_gather/cgroup" + // +kubebuilder:default="jobacct_gather/none" JobAcctGatherType *string `json:"jobAcctGatherType,omitempty"` // +kubebuilder:validation:Optional // +kubebuilder:default=0 diff --git a/config/crd/bases/slurm.nebius.ai_slurmclusters.yaml b/config/crd/bases/slurm.nebius.ai_slurmclusters.yaml index 4f845088e..2b3a3719c 100644 --- a/config/crd/bases/slurm.nebius.ai_slurmclusters.yaml +++ b/config/crd/bases/slurm.nebius.ai_slurmclusters.yaml @@ -8283,7 +8283,7 @@ spec: - DisableGPUAcct type: string jobAcctGatherType: - default: jobacct_gather/cgroup + default: jobacct_gather/none enum: - jobacct_gather/linux - jobacct_gather/cgroup diff --git a/helm/slurm-cluster/tests/README.md b/helm/slurm-cluster/tests/README.md index b719afcac..0fd9a6971 100644 --- a/helm/slurm-cluster/tests/README.md +++ b/helm/slurm-cluster/tests/README.md @@ -80,7 +80,7 @@ These tests verify the following kubebuilder default values: ### Accounting - `enabled: false` - `slurmConfig.accountingStorageTRES: "CPU,Mem,Node,VMem,Gres/gpu"` -- `slurmConfig.jobAcctGatherType: "jobacct_gather/cgroup"` +- `slurmConfig.jobAcctGatherType: "jobacct_gather/none"` - `slurmConfig.jobAcctGatherFrequency: 30` - `slurmConfig.priorityWeightAge: 0` - `slurmConfig.priorityWeightFairshare: 0` diff --git a/helm/slurm-cluster/tests/default-values_test.yaml b/helm/slurm-cluster/tests/default-values_test.yaml index e4444b1e3..03a7d3b7c 100644 --- a/helm/slurm-cluster/tests/default-values_test.yaml +++ b/helm/slurm-cluster/tests/default-values_test.yaml @@ -146,7 +146,7 @@ tests: value: "CPU,Mem,Node,VMem,Gres/gpu" - equal: path: spec.slurmNodes.accounting.slurmConfig.jobAcctGatherType - value: "jobacct_gather/cgroup" + value: "jobacct_gather/none" - equal: path: spec.slurmNodes.accounting.slurmConfig.jobAcctGatherFrequency value: 0 diff --git a/helm/slurm-cluster/values.yaml b/helm/slurm-cluster/values.yaml index 45f73c9e2..8b6f1a24b 100644 --- a/helm/slurm-cluster/values.yaml +++ b/helm/slurm-cluster/values.yaml @@ -234,7 +234,7 @@ slurmNodes: # hostUsers: false slurmConfig: accountingStorageTRES: "CPU,Mem,Node,VMem,Gres/gpu" - jobAcctGatherType: "jobacct_gather/cgroup" + jobAcctGatherType: "jobacct_gather/none" jobAcctGatherFrequency: 0 priorityWeightAge: 0 priorityWeightFairshare: 0 diff --git a/helm/soperator-crds/templates/slurmcluster-crd.yaml b/helm/soperator-crds/templates/slurmcluster-crd.yaml index 68fc6672b..c9fe1bf59 100644 --- a/helm/soperator-crds/templates/slurmcluster-crd.yaml +++ b/helm/soperator-crds/templates/slurmcluster-crd.yaml @@ -35567,7 +35567,7 @@ spec: - DisableGPUAcct type: string jobAcctGatherType: - default: jobacct_gather/cgroup + default: jobacct_gather/none enum: - jobacct_gather/linux - jobacct_gather/cgroup diff --git a/helm/soperator/crds/slurmcluster-crd.yaml b/helm/soperator/crds/slurmcluster-crd.yaml index 68fc6672b..c9fe1bf59 100644 --- a/helm/soperator/crds/slurmcluster-crd.yaml +++ b/helm/soperator/crds/slurmcluster-crd.yaml @@ -35567,7 +35567,7 @@ spec: - DisableGPUAcct type: string jobAcctGatherType: - default: jobacct_gather/cgroup + default: jobacct_gather/none enum: - jobacct_gather/linux - jobacct_gather/cgroup From 18101384357ae0035ece8fe1e0678b6ce8aa866a Mon Sep 17 00:00:00 2001 From: Dmitry Starov Date: Fri, 17 Jul 2026 16:40:52 +0200 Subject: [PATCH 2/5] [REF] Tidy up slurm.conf renderer --- internal/render/common/configmap.go | 56 ++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/internal/render/common/configmap.go b/internal/render/common/configmap.go index ff383386b..34889b0ef 100644 --- a/internal/render/common/configmap.go +++ b/internal/render/common/configmap.go @@ -262,11 +262,12 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile { controllerHostname := cluster.NodeController.StatefulSet.Name + "-0" res.AddProperty("SlurmctldHost", fmt.Sprintf("%s(%s)", controllerHostname, svcName)) } - res.AddComment("") + res.AddProperty("AuthType", "auth/"+consts.Munge) res.AddProperty("CredType", "cred/"+consts.Munge) res.AddComment("") + res.AddComment("SlurmConfig Spec") addSlurmConfigProperties(res, cluster.SlurmConfig) res.AddComment("") @@ -281,14 +282,18 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile { res.AddProperty("ProctrackType", "proctrack/cgroup") res.AddProperty("ReturnToService", 2) res.AddComment("") + res.AddProperty("SlurmctldPidFile", "/var/run/"+consts.SlurmctldName+".pid") res.AddProperty("SlurmctldPort", cluster.NodeController.ContainerSlurmctld.Port) + res.AddComment("") + // Slurm silently disables the metrics plugin if PrivateData is set in // the generated Slurm config. Don't add a PrivateData property anywhere in this file. if om := cluster.NodeController.OpenMetrics; om.Enabled == nil || *om.Enabled { res.AddProperty("MetricsType", "metrics/openmetrics") + res.AddComment("") } - res.AddComment("") + res.AddProperty("SlurmdPidFile", "/var/run/"+consts.SlurmdName+".pid") res.AddComment("") res.AddProperty("SlurmdSpoolDir", naming.BuildVolumeMountSpoolPath(consts.SlurmdName)) @@ -309,21 +314,23 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile { res.AddComment("") res.AddProperty("SchedulerParameters", "nohold_on_prolog_fail,extra_constraints,pack_serial_at_end,salloc_wait_nodes,sbatch_wait_nodes") res.AddComment("") - res.AddComment("HEALTH CHECKS") - res.AddComment("https://slurm.schedmd.com/slurm.conf.html#OPT_HealthCheckInterval") if cluster.HealthCheckConfig != nil { + res.AddComment("HEALTH CHECKS") + res.AddComment("https://slurm.schedmd.com/slurm.conf.html#OPT_HealthCheckInterval") + res.AddProperty("HealthCheckInterval", cluster.HealthCheckConfig.HealthCheckInterval) res.AddProperty("HealthCheckProgram", cluster.HealthCheckConfig.HealthCheckProgram) - var states []string - for _, state := range cluster.HealthCheckConfig.HealthCheckNodeState { - states = append(states, state.State) + states := make([]string, len(cluster.HealthCheckConfig.HealthCheckNodeState)) + for i, state := range cluster.HealthCheckConfig.HealthCheckNodeState { + states[i] = state.State } res.AddProperty("HealthCheckNodeState", strings.Join(states, ",")) + + res.AddComment("") } - res.AddComment("") res.AddProperty("InactiveLimit", 0) res.AddProperty("KillOnBadExit", 1) res.AddProperty("KillWait", 180) @@ -332,19 +339,31 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile { res.AddProperty("SlurmdTimeout", 180) res.AddProperty("TCPTimeout", 15) res.AddProperty("WaitTime", 0) + res.AddComment("") + total := totalWorkerNodes(cluster) - connMax := max(int32(1024), nextPow2(total*2)) - if cluster.HasEphemeralNodes() { - res.AddProperty("SlurmctldParameters", fmt.Sprintf("conmgr_max_connections=%d,conmgr_threads=32,cloud_dns,idle_on_node_suspend", connMax)) - } else { - res.AddProperty("SlurmctldParameters", fmt.Sprintf("conmgr_max_connections=%d,conmgr_threads=32", connMax)) + + { + connMax := max(int32(1024), nextPow2(total*2)) + slurmCtldParams := []string{ + fmt.Sprintf("conmgr_max_connections=%d", connMax), + "conmgr_threads=32", + } + if cluster.HasEphemeralNodes() { + slurmCtldParams = append(slurmCtldParams, []string{ + "cloud_dns", + "idle_on_node_suspend", + }...) + } + res.AddProperty("SlurmctldParameters", strings.Join(slurmCtldParams, ",")) } + res.AddComment("") res.AddProperty("RebootProgram", "/opt/bin/slurm/reboot.sh") res.AddProperty("ResumeTimeout", 1800) + res.AddComment("") // Power management for ephemeral nodes - res.AddComment("") res.AddComment("POWER MANAGEMENT (ephemeral nodes)") res.AddProperty("ResumeProgram", "/opt/soperator/bin/power_resume.sh") res.AddProperty("SuspendProgram", "/opt/soperator/bin/power_suspend.sh") @@ -355,13 +374,14 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile { if suspendExcNodes := buildSuspendExcNodes(cluster); suspendExcNodes != "" { res.AddProperty("SuspendExcNodes", suspendExcNodes) } - res.AddComment("") + res.AddComment("SCHEDULING") res.AddProperty("SchedulerType", "sched/backfill") res.AddProperty("SelectType", "select/cons_tres") res.AddProperty("SelectTypeParameters", "CR_Core_Memory,CR_CORE_DEFAULT_DIST_BLOCK") res.AddComment("") + res.AddComment("LOGGING") res.AddProperty("SlurmctldDebug", consts.SlurmDefaultDebugLevel) res.AddProperty("SlurmctldLogFile", consts.SlurmLogFile) @@ -369,6 +389,7 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile { res.AddProperty("SlurmdLogFile", consts.SlurmLogFile) res.AddProperty("DebugFlags", "Script,Power") res.AddComment("") + res.AddComment("COMPUTE NODES") res.AddComment("We're using the \"dynamic nodes\" feature: https://slurm.schedmd.com/dynamic_nodes.html") res.AddProperty("MaxNodeCount", max(int32(1024), nextPow2(total*2))) @@ -376,8 +397,8 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile { res.AddProperty("JobRequeue", 1) res.AddProperty("PreemptMode", "REQUEUE") res.AddProperty("PreemptType", "preempt/partition_prio") - res.AddComment("Partition Configuration") + res.AddComment("Partition Configuration") switch cluster.PartitionConfiguration.ConfigType { case slurmv1.PartitionConfigTypeCustom: for _, l := range cluster.PartitionConfiguration.RawConfig { @@ -387,6 +408,7 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile { res.AddProperty("PartitionName", clearLine) } } + case slurmv1.PartitionConfigTypeStructured: AddNodesToSlurmConfig(res, cluster) AddPartitionsToSlurmConfig(res, cluster) @@ -396,9 +418,9 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile { res.AddProperty("PartitionName", "main Nodes=ALL Default=YES PriorityTier=10 MaxTime=INFINITE State=UP OverSubscribe=YES") res.AddProperty("PartitionName", "hidden Nodes=ALL Default=NO PriorityTier=10 PreemptMode=OFF Hidden=YES MaxTime=INFINITE State=UP OverSubscribe=YES") } + res.AddComment("") if cluster.NodeAccounting.Enabled { - res.AddComment("") res.AddComment("ACCOUNTING") res.AddProperty("AccountingStorageType", "accounting_storage/slurmdbd") res.AddProperty("AccountingStorageHost", naming.BuildServiceFQDN( From 9dce5af07fb0c9f1e8712b8553d3dcabc816f077 Mon Sep 17 00:00:00 2001 From: Dmitry Starov Date: Fri, 17 Jul 2026 17:18:40 +0200 Subject: [PATCH 3/5] [ADD] Improve SchedulerParameters --- internal/render/common/configmap.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/internal/render/common/configmap.go b/internal/render/common/configmap.go index 34889b0ef..18b62113e 100644 --- a/internal/render/common/configmap.go +++ b/internal/render/common/configmap.go @@ -312,8 +312,6 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile { res.AddComment("") res.AddProperty("PropagateResourceLimits", "NONE") // Don't propagate ulimits from the login node by default res.AddComment("") - res.AddProperty("SchedulerParameters", "nohold_on_prolog_fail,extra_constraints,pack_serial_at_end,salloc_wait_nodes,sbatch_wait_nodes") - res.AddComment("") if cluster.HealthCheckConfig != nil { res.AddComment("HEALTH CHECKS") @@ -378,6 +376,23 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile { res.AddComment("SCHEDULING") res.AddProperty("SchedulerType", "sched/backfill") + res.AddProperty("SchedulerParameters", + strings.Join([]string{ + "nohold_on_prolog_fail", // allows jobs to be dispatched to another hosts during requeue + "extra_constraints", // enables filtering nodes with the --extra option + "pack_serial_at_end", // reduces resource fragmentation for some workloads with select/cons_tres plugin + "salloc_wait_nodes", // wait until all allocated nodes are ready for use in salloc + "sbatch_wait_nodes", // wait until all allocated nodes are ready for use in sbatch + "defer_batch", // avoid attempting to schedule each batch job individually at job submit time + "max_rpc_cnt=100", // maximum number of threads before deferring job scheduling + // sched/backfill + "bf_continue", // resume mid-cycle instead of restarting from the top after releasing the lock + "bf_max_time=60", // maximum time the backfill scheduler can spend before discontinuing (s) + "bf_max_job_test=1000", // maximum number of jobs to attempt scheduling for + "bf_max_job_part=300", // maximum number of jobs per partition to attempt starting with the scheduler + "bf_running_job_reserve", // creates backfill reservations for jobs running on whole nodes + }, ","), + ) res.AddProperty("SelectType", "select/cons_tres") res.AddProperty("SelectTypeParameters", "CR_Core_Memory,CR_CORE_DEFAULT_DIST_BLOCK") res.AddComment("") From f072fe7763e86c1e2bd525856df2527b9565d437 Mon Sep 17 00:00:00 2001 From: Dmitry Starov Date: Fri, 17 Jul 2026 17:46:42 +0200 Subject: [PATCH 4/5] [ADD] Improve SlurmctldParameters --- internal/render/common/configmap.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/render/common/configmap.go b/internal/render/common/configmap.go index 18b62113e..66eaed987 100644 --- a/internal/render/common/configmap.go +++ b/internal/render/common/configmap.go @@ -344,17 +344,20 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile { { connMax := max(int32(1024), nextPow2(total*2)) slurmCtldParams := []string{ - fmt.Sprintf("conmgr_max_connections=%d", connMax), - "conmgr_threads=32", + fmt.Sprintf("conmgr_max_connections=%d", connMax), // maximum number of connections to be processed at any given time + "conmgr_threads=32", // number of threads in thread pool used for connections on the listening sockets + "validate_nodeaddr_threads=32", // permit concurrent node address validation during startup + "enable_stepmgr", // enable job steps to be managed by a single extern job-associated slurmstepd } if cluster.HasEphemeralNodes() { slurmCtldParams = append(slurmCtldParams, []string{ - "cloud_dns", - "idle_on_node_suspend", + "cloud_dns", // avoid informing ip addresses of cloud nodes to client commands + "idle_on_node_suspend", // allow suspended nodes to be resumed by marking them idle }...) } res.AddProperty("SlurmctldParameters", strings.Join(slurmCtldParams, ",")) } + res.AddProperty("PrologFlags", "contain") // use ProcTrack for creating a job container on all allocated workers during job allocation res.AddComment("") res.AddProperty("RebootProgram", "/opt/bin/slurm/reboot.sh") From aa92176c108fba34209dcf6ebc8ed83ba63110f7 Mon Sep 17 00:00:00 2001 From: Dmitry Starov Date: Fri, 17 Jul 2026 18:15:55 +0200 Subject: [PATCH 5/5] [ADD] Reduce default MinJobAge to 30 minutes --- api/v1/slurmcluster_types.go | 4 ++-- config/crd/bases/slurm.nebius.ai_slurmclusters.yaml | 4 ++-- helm/slurm-cluster/tests/README.md | 2 +- helm/slurm-cluster/tests/default-values_test.yaml | 2 +- helm/slurm-cluster/values.yaml | 2 +- helm/soperator-crds/templates/slurmcluster-crd.yaml | 4 ++-- helm/soperator/crds/slurmcluster-crd.yaml | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/api/v1/slurmcluster_types.go b/api/v1/slurmcluster_types.go index 4ecd29c23..34fca6407 100644 --- a/api/v1/slurmcluster_types.go +++ b/api/v1/slurmcluster_types.go @@ -66,7 +66,7 @@ type SlurmClusterSpec struct { // SlurmConfig represents the Slurm configuration in slurm.conf. Not all options are supported. // // +kubebuilder:validation:Optional - // +kubebuilder:default={defMemPerNode: 0, defCpuPerGPU: 4, completeWait: 5, epilog: "", prolog: "", taskProlog: "", maxJobCount: 20000, minJobAge: 3600, messageTimeout: 60} + // +kubebuilder:default={defMemPerNode: 0, defCpuPerGPU: 4, completeWait: 5, epilog: "", prolog: "", taskProlog: "", maxJobCount: 20000, minJobAge: 1800, messageTimeout: 60} SlurmConfig SlurmConfig `json:"slurmConfig,omitempty"` // Topology contains topology-related parameters for Slurm. @@ -170,7 +170,7 @@ type SlurmConfig struct { // Don't remove jobs from controller memory after some time // // +kubebuilder:validation:Optional - // +kubebuilder:default=3600 + // +kubebuilder:default=1800 MinJobAge *int32 `json:"minJobAge,omitempty"` // MessageTimeout specifies the permitted time for a round-trip communication to complete in seconds. // See https://slurm.schedmd.com/slurm.conf.html#OPT_MessageTimeout. diff --git a/config/crd/bases/slurm.nebius.ai_slurmclusters.yaml b/config/crd/bases/slurm.nebius.ai_slurmclusters.yaml index 2b3a3719c..110be0024 100644 --- a/config/crd/bases/slurm.nebius.ai_slurmclusters.yaml +++ b/config/crd/bases/slurm.nebius.ai_slurmclusters.yaml @@ -3806,7 +3806,7 @@ spec: epilog: "" maxJobCount: 20000 messageTimeout: 60 - minJobAge: 3600 + minJobAge: 1800 prolog: "" taskProlog: "" description: SlurmConfig represents the Slurm configuration in slurm.conf. @@ -3847,7 +3847,7 @@ spec: format: int32 type: integer minJobAge: - default: 3600 + default: 1800 description: Don't remove jobs from controller memory after some time format: int32 diff --git a/helm/slurm-cluster/tests/README.md b/helm/slurm-cluster/tests/README.md index 0fd9a6971..523ba0a00 100644 --- a/helm/slurm-cluster/tests/README.md +++ b/helm/slurm-cluster/tests/README.md @@ -42,7 +42,7 @@ These tests verify the following kubebuilder default values: - `taskProlog: ""` - `taskPluginParam: ""` - `maxJobCount: 20000` -- `minJobAge: 3600` +- `minJobAge: 1800` - `messageTimeout: 60` - `topologyPlugin: "topology/tree"` - `topologyParam: "SwitchAsNodeRank"` diff --git a/helm/slurm-cluster/tests/default-values_test.yaml b/helm/slurm-cluster/tests/default-values_test.yaml index 03a7d3b7c..8c3946b0f 100644 --- a/helm/slurm-cluster/tests/default-values_test.yaml +++ b/helm/slurm-cluster/tests/default-values_test.yaml @@ -42,7 +42,7 @@ tests: value: 20000 - equal: path: spec.slurmConfig.minJobAge - value: 3600 + value: 1800 - equal: path: spec.slurmConfig.messageTimeout value: 60 diff --git a/helm/slurm-cluster/values.yaml b/helm/slurm-cluster/values.yaml index 8b6f1a24b..551844cdf 100644 --- a/helm/slurm-cluster/values.yaml +++ b/helm/slurm-cluster/values.yaml @@ -153,7 +153,7 @@ slurmConfig: epilog: /opt/slurm_scripts/epilog.sh taskPluginParam: "" maxJobCount: 20000 - minJobAge: 3600 + minJobAge: 1800 messageTimeout: 60 topologyPlugin: "topology/tree" topologyParam: "SwitchAsNodeRank" diff --git a/helm/soperator-crds/templates/slurmcluster-crd.yaml b/helm/soperator-crds/templates/slurmcluster-crd.yaml index c9fe1bf59..6b07761df 100644 --- a/helm/soperator-crds/templates/slurmcluster-crd.yaml +++ b/helm/soperator-crds/templates/slurmcluster-crd.yaml @@ -31090,7 +31090,7 @@ spec: epilog: "" maxJobCount: 20000 messageTimeout: 60 - minJobAge: 3600 + minJobAge: 1800 prolog: "" taskProlog: "" description: SlurmConfig represents the Slurm configuration in slurm.conf. @@ -31131,7 +31131,7 @@ spec: format: int32 type: integer minJobAge: - default: 3600 + default: 1800 description: Don't remove jobs from controller memory after some time format: int32 diff --git a/helm/soperator/crds/slurmcluster-crd.yaml b/helm/soperator/crds/slurmcluster-crd.yaml index c9fe1bf59..6b07761df 100644 --- a/helm/soperator/crds/slurmcluster-crd.yaml +++ b/helm/soperator/crds/slurmcluster-crd.yaml @@ -31090,7 +31090,7 @@ spec: epilog: "" maxJobCount: 20000 messageTimeout: 60 - minJobAge: 3600 + minJobAge: 1800 prolog: "" taskProlog: "" description: SlurmConfig represents the Slurm configuration in slurm.conf. @@ -31131,7 +31131,7 @@ spec: format: int32 type: integer minJobAge: - default: 3600 + default: 1800 description: Don't remove jobs from controller memory after some time format: int32