feat: add annotations flag for kubernetes jobs

charts/libs/runner/Chart.yaml

@@ -15,7 +15,7 @@ type: library
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.1.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/libs/runner/README.md

@@ -1,6 +1,6 @@
 # runner
 
-![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square)
+![Version: 0.1.1](https://img.shields.io/badge/Version-0.1.1-informational?style=flat-square) ![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square)
 
 A Helm chart library for otf runner resources and configuration.
 
@@ -14,6 +14,7 @@ A Helm chart library for otf runner resources and configuration.
 | cacheVolume.storageClass | string | `nil` | Persistent volume storage class. # If defined, storageClassName: <storageClass> # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is # set, choosing the default provisioner. |
 | concurrency | int | `nil` | Set the number of runs that can be processed concurrently. See [docs](https://docs.otf.ninja/config/flags/#-concurrency). |
 | executor | string | `""` | The executor to use. See [docs](https://docs.otf.ninja/config/flags/#-executor) |
+| kubernetesAnnotations | list | `[]` | Set additional annotations on the pods created for kubernetes jobs. Name and value are separated by an equals sign, e.g. `cluster-autoscaler.kubernetes.io/safe-to-evict=false`. |
 | kubernetesJobImage | string | `""` | Set the kubernetes job image. Defaults to `leg100/otf-job` tagged with the current OTF version. See [docs](https://docs.otf.ninja/config/flags/#-kubernetes-job-image). |
 | kubernetesLabels | list | `[]` | Set additional labels on kubernetes jobs. Name and value are separated by an equals sign, e.g. `foo=bar`. |
 | kubernetesLimitCPU | string | `nil` | Set a CPU limit for kubernetes jobs. |

charts/libs/runner/templates/_envs.yaml

@@ -16,6 +16,10 @@
 - name: OTF_KUBERNETES_LABELS
   value: {{ . | join "," }}
 {{- end }}
+{{- with .kubernetesAnnotations }}
+- name: OTF_KUBERNETES_ANNOTATIONS
+  value: {{ . | join "," }}
+{{- end }}
 {{- with .kubernetesRequestCPU }}
 - name: OTF_KUBERNETES_REQUEST_CPU
   value: {{ . }}

charts/libs/runner/values.yaml

@@ -12,6 +12,8 @@ kubernetesLimitCPU: ~
 kubernetesLimitMemory: ~
 # -- Set additional labels on kubernetes jobs. Name and value are separated by an equals sign, e.g. `foo=bar`.
 kubernetesLabels: []
+# -- Set additional annotations on the pods created for kubernetes jobs. Name and value are separated by an equals sign, e.g. `cluster-autoscaler.kubernetes.io/safe-to-evict=false`.
+kubernetesAnnotations: []
 # -- Set the kubernetes job image. Defaults to `leg100/otf-job` tagged with the current OTF version. See [docs](https://docs.otf.ninja/config/flags/#-kubernetes-job-image).
 kubernetesJobImage: ""
 # -- Delete finished kubernetes jobs after this duration.

charts/otf-agent/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: otf-agent
 description: Helm chart for the OTF agent
 type: application
-version: 0.1.33
+version: 0.1.34
 appVersion: 0.6.3
 maintainers:
   - name: Louis Garman

charts/otf-agent/README.md

@@ -1,6 +1,6 @@
 # Helm Chart for `otf-agent`
 
-![Version: 0.1.33](https://img.shields.io/badge/Version-0.1.33-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.3](https://img.shields.io/badge/AppVersion-0.6.3-informational?style=flat-square)
+![Version: 0.1.34](https://img.shields.io/badge/Version-0.1.34-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.3](https://img.shields.io/badge/AppVersion-0.6.3-informational?style=flat-square)
 
 Installs the [otf agent](https://docs.otf.ninja/runners/).
 
@@ -89,6 +89,7 @@ address=10.244.0.18 agent.pool_id=apool-5b90443ed82ef769
 | runner.cacheVolume.storageClass | string | `nil` | Persistent volume storage class. # If defined, storageClassName: <storageClass> # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is # set, choosing the default provisioner. |
 | runner.concurrency | int | `nil` | Set the number of runs that can be processed concurrently. See [docs](https://docs.otf.ninja/config/flags/#-concurrency). |
 | runner.executor | string | `""` | The executor to use. See [docs](https://docs.otf.ninja/config/flags/#-executor) |
+| runner.kubernetesAnnotations | list | `[]` | Set additional annotations on the pods created for kubernetes jobs. Name and value are separated by an equals sign, e.g. `cluster-autoscaler.kubernetes.io/safe-to-evict=false`. |
 | runner.kubernetesJobImage | string | `""` | Set the kubernetes job image. Defaults to `leg100/otf-job` tagged with the current OTF version. See [docs](https://docs.otf.ninja/config/flags/#-kubernetes-job-image). |
 | runner.kubernetesLabels | list | `[]` | Set additional labels on kubernetes jobs. Name and value are separated by an equals sign, e.g. `foo=bar`. |
 | runner.kubernetesLimitCPU | string | `nil` | Set a CPU limit for kubernetes jobs. |

charts/otfd/Chart.yaml

@@ -13,7 +13,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.2
+version: 0.4.3
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.

charts/otfd/README.md

@@ -1,6 +1,6 @@
 # Helm chart for `otfd`
 
-![Version: 0.4.2](https://img.shields.io/badge/Version-0.4.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.3](https://img.shields.io/badge/AppVersion-0.6.3-informational?style=flat-square)
+![Version: 0.4.3](https://img.shields.io/badge/Version-0.4.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.3](https://img.shields.io/badge/AppVersion-0.6.3-informational?style=flat-square)
 
 Installs the [otf](https://github.com/leg100/otf) daemon.
 
@@ -111,6 +111,7 @@ Note: you should only use this for testing purposes.
 | runner.cacheVolume.storageClass | string | `nil` | Persistent volume storage class. # If defined, storageClassName: <storageClass> # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is # set, choosing the default provisioner. |
 | runner.concurrency | int | `nil` | Set the number of runs that can be processed concurrently. See [docs](https://docs.otf.ninja/config/flags/#-concurrency). |
 | runner.executor | string | `""` | The executor to use. See [docs](https://docs.otf.ninja/config/flags/#-executor) |
+| runner.kubernetesAnnotations | list | `[]` | Set additional annotations on the pods created for kubernetes jobs. Name and value are separated by an equals sign, e.g. `cluster-autoscaler.kubernetes.io/safe-to-evict=false`. |
 | runner.kubernetesJobImage | string | `""` | Set the kubernetes job image. Defaults to `leg100/otf-job` tagged with the current OTF version. See [docs](https://docs.otf.ninja/config/flags/#-kubernetes-job-image). |
 | runner.kubernetesLabels | list | `[]` | Set additional labels on kubernetes jobs. Name and value are separated by an equals sign, e.g. `foo=bar`. |
 | runner.kubernetesLimitCPU | string | `nil` | Set a CPU limit for kubernetes jobs. |

docs/docs/config/flags.md

@@ -170,6 +170,13 @@ Set a memory limit for kubernetes jobs.
 
 Set additional labels on kubernetes jobs. Name and value are separated by an equals sign, e.g. `foo=bar`.
 
+## `--kubernetes-annotations`
+
+* System: `otfd`, `otf-agent`
+* Default: `""`
+
+Set additional annotations on the pods created for kubernetes jobs. Name and value are separated by an equals sign, e.g. `cluster-autoscaler.kubernetes.io/safe-to-evict=false`. Specify multiple annotations separated by a comma. Use this to set the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-types-of-pods-can-prevent-ca-from-removing-a-node) `cluster-autoscaler.kubernetes.io/safe-to-evict` annotation so that in-progress runs are not disrupted by node scale-down.
+
 ## `--kubernetes-ttl-after-finish`
 
 * System: `otfd`, `otf-agent`

docs/docs/executors.md

@@ -26,6 +26,7 @@ This executor is only functional when `otfd` or `otf-agent` is deployed via the
 There are a number of flags that customise the jobs:
 
 * [`--kubernetes-job-image`](config/flags.md#-kubernetes-job-image)
+* [`--kubernetes-annotations`](config/flags.md#-kubernetes-annotations)
 * [`--kubernetes-request-cpu`](config/flags.md#-kubernetes-request-cpu)
 * [`--kubernetes-request-memory`](config/flags.md#-kubernetes-request-memory)
 * [`--kubernetes-ttl-after-finish`](config/flags.md#-kubernetes-ttl-after-finish)

internal/runner/executor_kube.go

@@ -87,6 +87,7 @@ type kubeConfig struct {
 	limitCPU      *k8sresource.Quantity
 	limitMemory   *k8sresource.Quantity
 	labels        map[string]string
+	annotations   map[string]string
 
 	flags kubeConfigFlags
 }
@@ -95,6 +96,7 @@ type kubeConfig struct {
 // used directly by the kubernetes executor.
 type kubeConfigFlags struct {
 	Labels        []string
+	Annotations   []string
 	RequestCPU    string
 	RequestMemory string
 	LimitCPU      string
@@ -109,6 +111,7 @@ func registerKubeFlags(flags *pflag.FlagSet, cfg *kubeConfig) {
 	flags.StringVar(&cfg.flags.LimitCPU, "kubernetes-limit-cpu", cfg.flags.LimitCPU, "CPU limit for kubernetes job.")
 	flags.StringVar(&cfg.flags.LimitMemory, "kubernetes-limit-memory", cfg.flags.LimitMemory, "Memory limit for kubernetes job.")
 	flags.StringSliceVar(&cfg.flags.Labels, "kubernetes-labels", cfg.flags.Labels, "Set additional labels on kubernetes jobs. Name and value are separated by an equals sign, e.g. `foo=bar`.")
+	flags.StringSliceVar(&cfg.flags.Annotations, "kubernetes-annotations", cfg.flags.Annotations, "Set additional annotations on the pods created for kubernetes jobs. Name and value are separated by an equals sign, e.g. `cluster-autoscaler.kubernetes.io/safe-to-evict=false`.")
 }
 
 type kubeExecutor struct {
@@ -177,6 +180,15 @@ func newKubeExecutor(
 		executor.Config.labels[k] = v
 	}
 
+	executor.Config.annotations = make(map[string]string)
+	for _, annotation := range kubeConfig.flags.Annotations {
+		k, v, ok := strings.Cut(annotation, "=")
+		if !ok {
+			return nil, fmt.Errorf("invalid annotation: must be in format name=value")
+		}
+		executor.Config.annotations[k] = v
+	}
+
 	// assume running in-cluster; otherwise use config path
 	config, err := rest.InClusterConfig()
 	if errors.Is(err, rest.ErrNotInCluster) {
@@ -210,6 +222,9 @@ func (s *kubeExecutor) SpawnOperation(ctx context.Context, _ *errgroup.Group, jo
 	}
 	maps.Copy(labels, s.Config.labels)
 
+	annotations := make(map[string]string, len(s.Config.annotations))
+	maps.Copy(annotations, s.Config.annotations)
+
 	const (
 		cacheVolumeName   = "cache"
 		jobTokenSecretKey = "jobToken"
@@ -260,7 +275,8 @@ func (s *kubeExecutor) SpawnOperation(ctx context.Context, _ *errgroup.Group, jo
 			TTLSecondsAfterFinished: new(int32(s.Config.TTLAfterFinish.Seconds())),
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
-					Labels: labels,
+					Labels:      labels,
+					Annotations: annotations,
 				},
 				Spec: corev1.PodSpec{
 					ServiceAccountName: s.Config.ServiceAccount,

internal/runner/executor_kube_test.go

@@ -74,11 +74,40 @@ func TestNewKubeExecutor(t *testing.T) {
 		)
 		assert.Error(t, err)
 	})
+
+	t.Run("with annotations", func(t *testing.T) {
+		cfg := defaultKubeConfig
+		cfg.flags.Annotations = []string{"cluster-autoscaler.kubernetes.io/safe-to-evict=false", "coo=boo"}
+
+		executor, err := newKubeExecutor(
+			logr.Discard(),
+			defaultOperationConfig(),
+			cfg,
+		)
+		require.NoError(t, err)
+		assert.Equal(t, map[string]string{
+			"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
+			"coo": "boo",
+		}, executor.Config.annotations)
+	})
+
+	t.Run("with invalid annotations", func(t *testing.T) {
+		cfg := defaultKubeConfig
+		cfg.flags.Annotations = []string{"foobar", "cooboo"}
+
+		_, err := newKubeExecutor(
+			logr.Discard(),
+			defaultOperationConfig(),
+			cfg,
+		)
+		assert.Error(t, err)
+	})
 }
 
 func TestKubeExecutor_SpawnOperation(t *testing.T) {
 	cfg := defaultKubeConfig
 	cfg.flags.Labels = []string{"foo=bar"}
+	cfg.flags.Annotations = []string{"cluster-autoscaler.kubernetes.io/safe-to-evict=false"}
 	cfg.flags.LimitCPU = "3000m"
 	cfg.flags.LimitMemory = "512Mi"
 
@@ -123,6 +152,13 @@ func TestKubeExecutor_SpawnOperation(t *testing.T) {
 	assert.Equal(t, wantLabels, jobsClient.job.Labels)
 	assert.Equal(t, wantLabels, secretsClient.secret.Labels)
 	assert.Equal(t, map[string]string{"jobToken": "token"}, secretsClient.secret.StringData)
+
+	// Annotations are set on the pods (via the pod template), not on the job
+	// or secret.
+	assert.Equal(t,
+		map[string]string{"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"},
+		jobsClient.job.Spec.Template.Annotations,
+	)
 }
 
 type fakeSecretsClient struct {