SCHED-1214: Tune slurm.conf

api/v1/slurmcluster_types.go

@@ -66,7 +66,7 @@ type SlurmClusterSpec struct {
 	// SlurmConfig represents the Slurm configuration in slurm.conf. Not all options are supported.
 	//
 	// +kubebuilder:validation:Optional
-	// +kubebuilder:default={defMemPerNode: 0, defCpuPerGPU: 4, completeWait: 5, epilog: "", prolog: "", taskProlog: "", maxJobCount: 20000, minJobAge: 3600, messageTimeout: 60}
+	// +kubebuilder:default={defMemPerNode: 0, defCpuPerGPU: 4, completeWait: 5, epilog: "", prolog: "", taskProlog: "", maxJobCount: 20000, minJobAge: 1800, messageTimeout: 60}
 	SlurmConfig SlurmConfig `json:"slurmConfig,omitempty"`
 
 	// Topology contains topology-related parameters for Slurm.
@@ -170,7 +170,7 @@ type SlurmConfig struct {
 	// Don't remove jobs from controller memory after some time
 	//
 	// +kubebuilder:validation:Optional
-	// +kubebuilder:default=3600
+	// +kubebuilder:default=1800
 	MinJobAge *int32 `json:"minJobAge,omitempty"`
 	// MessageTimeout specifies the permitted time for a round-trip communication to complete in seconds.
 	// See https://slurm.schedmd.com/slurm.conf.html#OPT_MessageTimeout.
@@ -884,7 +884,7 @@ type AccountingSlurmConf struct {
 	AcctGatherProfileType *string `json:"acctGatherProfileType,omitempty"`
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:validation:Enum="jobacct_gather/linux";"jobacct_gather/cgroup";"jobacct_gather/none"
-	// +kubebuilder:default="jobacct_gather/cgroup"
+	// +kubebuilder:default="jobacct_gather/none"
 	JobAcctGatherType *string `json:"jobAcctGatherType,omitempty"`
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:default=0

config/crd/bases/slurm.nebius.ai_slurmclusters.yaml

@@ -3806,7 +3806,7 @@ spec:
                   epilog: ""
                   maxJobCount: 20000
                   messageTimeout: 60
-                  minJobAge: 3600
+                  minJobAge: 1800
                   prolog: ""
                   taskProlog: ""
                 description: SlurmConfig represents the Slurm configuration in slurm.conf.
@@ -3847,7 +3847,7 @@ spec:
                     format: int32
                     type: integer
                   minJobAge:
-                    default: 3600
+                    default: 1800
                     description: Don't remove jobs from controller memory after some
                       time
                     format: int32
@@ -8283,7 +8283,7 @@ spec:
                             - DisableGPUAcct
                             type: string
                           jobAcctGatherType:
-                            default: jobacct_gather/cgroup
+                            default: jobacct_gather/none
                             enum:
                             - jobacct_gather/linux
                             - jobacct_gather/cgroup

helm/slurm-cluster/tests/README.md

@@ -42,7 +42,7 @@ These tests verify the following kubebuilder default values:
 - `taskProlog: ""`
 - `taskPluginParam: ""`
 - `maxJobCount: 20000`
-- `minJobAge: 3600`
+- `minJobAge: 1800`
 - `messageTimeout: 60`
 - `topologyPlugin: "topology/tree"`
 - `topologyParam: "SwitchAsNodeRank"`
@@ -80,7 +80,7 @@ These tests verify the following kubebuilder default values:
 ### Accounting
 - `enabled: false`
 - `slurmConfig.accountingStorageTRES: "CPU,Mem,Node,VMem,Gres/gpu"`
-- `slurmConfig.jobAcctGatherType: "jobacct_gather/cgroup"`
+- `slurmConfig.jobAcctGatherType: "jobacct_gather/none"`
 - `slurmConfig.jobAcctGatherFrequency: 30`
 - `slurmConfig.priorityWeightAge: 0`
 - `slurmConfig.priorityWeightFairshare: 0`

helm/slurm-cluster/tests/default-values_test.yaml

@@ -42,7 +42,7 @@ tests:
           value: 20000
       - equal:
           path: spec.slurmConfig.minJobAge
-          value: 3600
+          value: 1800
       - equal:
           path: spec.slurmConfig.messageTimeout
           value: 60
@@ -146,7 +146,7 @@ tests:
           value: "CPU,Mem,Node,VMem,Gres/gpu"
       - equal:
           path: spec.slurmNodes.accounting.slurmConfig.jobAcctGatherType
-          value: "jobacct_gather/cgroup"
+          value: "jobacct_gather/none"
       - equal:
           path: spec.slurmNodes.accounting.slurmConfig.jobAcctGatherFrequency
           value: 0

helm/slurm-cluster/values.yaml

@@ -153,7 +153,7 @@ slurmConfig:
   epilog: /opt/slurm_scripts/epilog.sh
   taskPluginParam: ""
   maxJobCount: 20000
-  minJobAge: 3600
+  minJobAge: 1800
   messageTimeout: 60
   topologyPlugin: "topology/tree"
   topologyParam: "SwitchAsNodeRank"
@@ -234,7 +234,7 @@ slurmNodes:
     # hostUsers: false
     slurmConfig:
       accountingStorageTRES: "CPU,Mem,Node,VMem,Gres/gpu"
-      jobAcctGatherType: "jobacct_gather/cgroup"
+      jobAcctGatherType: "jobacct_gather/none"
       jobAcctGatherFrequency: 0
       priorityWeightAge: 0
       priorityWeightFairshare: 0

helm/soperator-crds/templates/slurmcluster-crd.yaml

@@ -31090,7 +31090,7 @@ spec:
                   epilog: ""
                   maxJobCount: 20000
                   messageTimeout: 60
-                  minJobAge: 3600
+                  minJobAge: 1800
                   prolog: ""
                   taskProlog: ""
                 description: SlurmConfig represents the Slurm configuration in slurm.conf.
@@ -31131,7 +31131,7 @@ spec:
                     format: int32
                     type: integer
                   minJobAge:
-                    default: 3600
+                    default: 1800
                     description: Don't remove jobs from controller memory after some
                       time
                     format: int32
@@ -35567,7 +35567,7 @@ spec:
                             - DisableGPUAcct
                             type: string
                           jobAcctGatherType:
-                            default: jobacct_gather/cgroup
+                            default: jobacct_gather/none
                             enum:
                             - jobacct_gather/linux
                             - jobacct_gather/cgroup

helm/soperator/crds/slurmcluster-crd.yaml

@@ -31090,7 +31090,7 @@ spec:
                   epilog: ""
                   maxJobCount: 20000
                   messageTimeout: 60
-                  minJobAge: 3600
+                  minJobAge: 1800
                   prolog: ""
                   taskProlog: ""
                 description: SlurmConfig represents the Slurm configuration in slurm.conf.
@@ -31131,7 +31131,7 @@ spec:
                     format: int32
                     type: integer
                   minJobAge:
-                    default: 3600
+                    default: 1800
                     description: Don't remove jobs from controller memory after some
                       time
                     format: int32
@@ -35567,7 +35567,7 @@ spec:
                             - DisableGPUAcct
                             type: string
                           jobAcctGatherType:
-                            default: jobacct_gather/cgroup
+                            default: jobacct_gather/none
                             enum:
                             - jobacct_gather/linux
                             - jobacct_gather/cgroup

internal/render/common/configmap.go

@@ -262,11 +262,12 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile {
 		controllerHostname := cluster.NodeController.StatefulSet.Name + "-0"
 		res.AddProperty("SlurmctldHost", fmt.Sprintf("%s(%s)", controllerHostname, svcName))
 	}
-
 	res.AddComment("")
+
 	res.AddProperty("AuthType", "auth/"+consts.Munge)
 	res.AddProperty("CredType", "cred/"+consts.Munge)
 	res.AddComment("")
+
 	res.AddComment("SlurmConfig Spec")
 	addSlurmConfigProperties(res, cluster.SlurmConfig)
 	res.AddComment("")
@@ -281,14 +282,18 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile {
 	res.AddProperty("ProctrackType", "proctrack/cgroup")
 	res.AddProperty("ReturnToService", 2)
 	res.AddComment("")
+
 	res.AddProperty("SlurmctldPidFile", "/var/run/"+consts.SlurmctldName+".pid")
 	res.AddProperty("SlurmctldPort", cluster.NodeController.ContainerSlurmctld.Port)
+	res.AddComment("")
+
 	// Slurm silently disables the metrics plugin if PrivateData is set in
 	// the generated Slurm config. Don't add a PrivateData property anywhere in this file.
 	if om := cluster.NodeController.OpenMetrics; om.Enabled == nil || *om.Enabled {
 		res.AddProperty("MetricsType", "metrics/openmetrics")
+		res.AddComment("")
 	}
-	res.AddComment("")
+
 	res.AddProperty("SlurmdPidFile", "/var/run/"+consts.SlurmdName+".pid")
 	res.AddComment("")
 	res.AddProperty("SlurmdSpoolDir", naming.BuildVolumeMountSpoolPath(consts.SlurmdName))
@@ -307,23 +312,23 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile {
 	res.AddComment("")
 	res.AddProperty("PropagateResourceLimits", "NONE") // Don't propagate ulimits from the login node by default
 	res.AddComment("")
-	res.AddProperty("SchedulerParameters", "nohold_on_prolog_fail,extra_constraints,pack_serial_at_end,salloc_wait_nodes,sbatch_wait_nodes")
-	res.AddComment("")
-	res.AddComment("HEALTH CHECKS")
-	res.AddComment("https://slurm.schedmd.com/slurm.conf.html#OPT_HealthCheckInterval")
 
 	if cluster.HealthCheckConfig != nil {
+		res.AddComment("HEALTH CHECKS")
+		res.AddComment("https://slurm.schedmd.com/slurm.conf.html#OPT_HealthCheckInterval")
+
 		res.AddProperty("HealthCheckInterval", cluster.HealthCheckConfig.HealthCheckInterval)
 		res.AddProperty("HealthCheckProgram", cluster.HealthCheckConfig.HealthCheckProgram)
 
-		var states []string
-		for _, state := range cluster.HealthCheckConfig.HealthCheckNodeState {
-			states = append(states, state.State)
+		states := make([]string, len(cluster.HealthCheckConfig.HealthCheckNodeState))
+		for i, state := range cluster.HealthCheckConfig.HealthCheckNodeState {
+			states[i] = state.State
 		}
 		res.AddProperty("HealthCheckNodeState", strings.Join(states, ","))
+
+		res.AddComment("")
 	}
 
-	res.AddComment("")
 	res.AddProperty("InactiveLimit", 0)
 	res.AddProperty("KillOnBadExit", 1)
 	res.AddProperty("KillWait", 180)
@@ -332,19 +337,34 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile {
 	res.AddProperty("SlurmdTimeout", 180)
 	res.AddProperty("TCPTimeout", 15)
 	res.AddProperty("WaitTime", 0)
+	res.AddComment("")
+
 	total := totalWorkerNodes(cluster)
-	connMax := max(int32(1024), nextPow2(total*2))
-	if cluster.HasEphemeralNodes() {
-		res.AddProperty("SlurmctldParameters", fmt.Sprintf("conmgr_max_connections=%d,conmgr_threads=32,cloud_dns,idle_on_node_suspend", connMax))
-	} else {
-		res.AddProperty("SlurmctldParameters", fmt.Sprintf("conmgr_max_connections=%d,conmgr_threads=32", connMax))
+
+	{
+		connMax := max(int32(1024), nextPow2(total*2))
+		slurmCtldParams := []string{
+			fmt.Sprintf("conmgr_max_connections=%d", connMax), // maximum number of connections to be processed at any given time
+			"conmgr_threads=32",            // number of threads in thread pool used for connections on the listening sockets
+			"validate_nodeaddr_threads=32", // permit concurrent node address validation during startup
+			"enable_stepmgr",               // enable job steps to be managed by a single extern job-associated slurmstepd
+		}
+		if cluster.HasEphemeralNodes() {
+			slurmCtldParams = append(slurmCtldParams, []string{
+				"cloud_dns",            // avoid informing ip addresses of cloud nodes to client commands
+				"idle_on_node_suspend", // allow suspended nodes to be resumed by marking them idle
+			}...)
+		}
+		res.AddProperty("SlurmctldParameters", strings.Join(slurmCtldParams, ","))
 	}
+	res.AddProperty("PrologFlags", "contain") // use ProcTrack for creating a job container on all allocated workers during job allocation
+	res.AddComment("")
 
 	res.AddProperty("RebootProgram", "/opt/bin/slurm/reboot.sh")
 	res.AddProperty("ResumeTimeout", 1800)
+	res.AddComment("")
 
 	// Power management for ephemeral nodes
-	res.AddComment("")
 	res.AddComment("POWER MANAGEMENT (ephemeral nodes)")
 	res.AddProperty("ResumeProgram", "/opt/soperator/bin/power_resume.sh")
 	res.AddProperty("SuspendProgram", "/opt/soperator/bin/power_suspend.sh")
@@ -355,29 +375,48 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile {
 	if suspendExcNodes := buildSuspendExcNodes(cluster); suspendExcNodes != "" {
 		res.AddProperty("SuspendExcNodes", suspendExcNodes)
 	}
-
 	res.AddComment("")
+
 	res.AddComment("SCHEDULING")
 	res.AddProperty("SchedulerType", "sched/backfill")
+	res.AddProperty("SchedulerParameters",
+		strings.Join([]string{
+			"nohold_on_prolog_fail", // allows jobs to be dispatched to another hosts during requeue
+			"extra_constraints",     // enables filtering nodes with the --extra option
+			"pack_serial_at_end",    // reduces resource fragmentation for some workloads with select/cons_tres plugin
+			"salloc_wait_nodes",     // wait until all allocated nodes are ready for use in salloc
+			"sbatch_wait_nodes",     // wait until all allocated nodes are ready for use in sbatch
+			"defer_batch",           // avoid attempting to schedule each batch job individually at job submit time
+			"max_rpc_cnt=100",       // maximum number of threads before deferring job scheduling
+			// sched/backfill
+			"bf_continue",            // resume mid-cycle instead of restarting from the top after releasing the lock
+			"bf_max_time=60",         // maximum time the backfill scheduler can spend before discontinuing (s)
+			"bf_max_job_test=1000",   // maximum number of jobs to attempt scheduling for
+			"bf_max_job_part=300",    // maximum number of jobs per partition to attempt starting with the scheduler
+			"bf_running_job_reserve", // creates backfill reservations for jobs running on whole nodes
+		}, ","),
+	)
 	res.AddProperty("SelectType", "select/cons_tres")
 	res.AddProperty("SelectTypeParameters", "CR_Core_Memory,CR_CORE_DEFAULT_DIST_BLOCK")
 	res.AddComment("")
+
 	res.AddComment("LOGGING")
 	res.AddProperty("SlurmctldDebug", consts.SlurmDefaultDebugLevel)
 	res.AddProperty("SlurmctldLogFile", consts.SlurmLogFile)
 	res.AddProperty("SlurmdDebug", consts.SlurmDefaultDebugLevel)
 	res.AddProperty("SlurmdLogFile", consts.SlurmLogFile)
 	res.AddProperty("DebugFlags", "Script,Power")
 	res.AddComment("")
+
 	res.AddComment("COMPUTE NODES")
 	res.AddComment("We're using the \"dynamic nodes\" feature: https://slurm.schedmd.com/dynamic_nodes.html")
 	res.AddProperty("MaxNodeCount", max(int32(1024), nextPow2(total*2)))
 	res.AddProperty("MaxArraySize", max(int32(1024), total*5))
 	res.AddProperty("JobRequeue", 1)
 	res.AddProperty("PreemptMode", "REQUEUE")
 	res.AddProperty("PreemptType", "preempt/partition_prio")
-	res.AddComment("Partition Configuration")
 
+	res.AddComment("Partition Configuration")
 	switch cluster.PartitionConfiguration.ConfigType {
 	case slurmv1.PartitionConfigTypeCustom:
 		for _, l := range cluster.PartitionConfiguration.RawConfig {
@@ -387,6 +426,7 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile {
 				res.AddProperty("PartitionName", clearLine)
 			}
 		}
+
 	case slurmv1.PartitionConfigTypeStructured:
 		AddNodesToSlurmConfig(res, cluster)
 		AddPartitionsToSlurmConfig(res, cluster)
@@ -396,9 +436,9 @@ func generateSlurmConfig(cluster *values.SlurmCluster) renderutils.ConfigFile {
 		res.AddProperty("PartitionName", "main Nodes=ALL Default=YES PriorityTier=10 MaxTime=INFINITE State=UP OverSubscribe=YES")
 		res.AddProperty("PartitionName", "hidden Nodes=ALL Default=NO PriorityTier=10 PreemptMode=OFF Hidden=YES MaxTime=INFINITE State=UP OverSubscribe=YES")
 	}
+	res.AddComment("")
 
 	if cluster.NodeAccounting.Enabled {
-		res.AddComment("")
 		res.AddComment("ACCOUNTING")
 		res.AddProperty("AccountingStorageType", "accounting_storage/slurmdbd")
 		res.AddProperty("AccountingStorageHost", naming.BuildServiceFQDN(