diff --git a/charts/libs/runner/Chart.yaml b/charts/libs/runner/Chart.yaml index 2812723f2..c07f8df00 100644 --- a/charts/libs/runner/Chart.yaml +++ b/charts/libs/runner/Chart.yaml @@ -15,7 +15,7 @@ type: library # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.1.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/libs/runner/README.md b/charts/libs/runner/README.md index 04bfead4b..e345daa1c 100644 --- a/charts/libs/runner/README.md +++ b/charts/libs/runner/README.md @@ -1,6 +1,6 @@ # runner -![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) +![Version: 0.1.1](https://img.shields.io/badge/Version-0.1.1-informational?style=flat-square) ![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) A Helm chart library for otf runner resources and configuration. @@ -14,6 +14,7 @@ A Helm chart library for otf runner resources and configuration. | cacheVolume.storageClass | string | `nil` | Persistent volume storage class. # If defined, storageClassName: # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is # set, choosing the default provisioner. | | concurrency | int | `nil` | Set the number of runs that can be processed concurrently. See [docs](https://docs.otf.ninja/config/flags/#-concurrency). | | executor | string | `""` | The executor to use. See [docs](https://docs.otf.ninja/config/flags/#-executor) | +| kubernetesAnnotations | list | `[]` | Set additional annotations on the pods created for kubernetes jobs. Name and value are separated by an equals sign, e.g. `cluster-autoscaler.kubernetes.io/safe-to-evict=false`. | | kubernetesJobImage | string | `""` | Set the kubernetes job image. Defaults to `leg100/otf-job` tagged with the current OTF version. See [docs](https://docs.otf.ninja/config/flags/#-kubernetes-job-image). | | kubernetesLabels | list | `[]` | Set additional labels on kubernetes jobs. Name and value are separated by an equals sign, e.g. `foo=bar`. | | kubernetesLimitCPU | string | `nil` | Set a CPU limit for kubernetes jobs. | diff --git a/charts/libs/runner/templates/_envs.yaml b/charts/libs/runner/templates/_envs.yaml index bdfdbcf09..aeeed762a 100644 --- a/charts/libs/runner/templates/_envs.yaml +++ b/charts/libs/runner/templates/_envs.yaml @@ -16,6 +16,10 @@ - name: OTF_KUBERNETES_LABELS value: {{ . | join "," }} {{- end }} +{{- with .kubernetesAnnotations }} +- name: OTF_KUBERNETES_ANNOTATIONS + value: {{ . | join "," }} +{{- end }} {{- with .kubernetesRequestCPU }} - name: OTF_KUBERNETES_REQUEST_CPU value: {{ . }} diff --git a/charts/libs/runner/values.yaml b/charts/libs/runner/values.yaml index 0c5a76cfb..2dfe157a7 100644 --- a/charts/libs/runner/values.yaml +++ b/charts/libs/runner/values.yaml @@ -12,6 +12,8 @@ kubernetesLimitCPU: ~ kubernetesLimitMemory: ~ # -- Set additional labels on kubernetes jobs. Name and value are separated by an equals sign, e.g. `foo=bar`. kubernetesLabels: [] +# -- Set additional annotations on the pods created for kubernetes jobs. Name and value are separated by an equals sign, e.g. `cluster-autoscaler.kubernetes.io/safe-to-evict=false`. +kubernetesAnnotations: [] # -- Set the kubernetes job image. Defaults to `leg100/otf-job` tagged with the current OTF version. See [docs](https://docs.otf.ninja/config/flags/#-kubernetes-job-image). kubernetesJobImage: "" # -- Delete finished kubernetes jobs after this duration. diff --git a/charts/otf-agent/Chart.yaml b/charts/otf-agent/Chart.yaml index 854c39ea7..8a99215ad 100644 --- a/charts/otf-agent/Chart.yaml +++ b/charts/otf-agent/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: otf-agent description: Helm chart for the OTF agent type: application -version: 0.1.33 +version: 0.1.34 appVersion: 0.6.3 maintainers: - name: Louis Garman diff --git a/charts/otf-agent/README.md b/charts/otf-agent/README.md index 314c94ff8..6bec0576b 100644 --- a/charts/otf-agent/README.md +++ b/charts/otf-agent/README.md @@ -1,6 +1,6 @@ # Helm Chart for `otf-agent` -![Version: 0.1.33](https://img.shields.io/badge/Version-0.1.33-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.3](https://img.shields.io/badge/AppVersion-0.6.3-informational?style=flat-square) +![Version: 0.1.34](https://img.shields.io/badge/Version-0.1.34-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.3](https://img.shields.io/badge/AppVersion-0.6.3-informational?style=flat-square) Installs the [otf agent](https://docs.otf.ninja/runners/). @@ -89,6 +89,7 @@ address=10.244.0.18 agent.pool_id=apool-5b90443ed82ef769 | runner.cacheVolume.storageClass | string | `nil` | Persistent volume storage class. # If defined, storageClassName: # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is # set, choosing the default provisioner. | | runner.concurrency | int | `nil` | Set the number of runs that can be processed concurrently. See [docs](https://docs.otf.ninja/config/flags/#-concurrency). | | runner.executor | string | `""` | The executor to use. See [docs](https://docs.otf.ninja/config/flags/#-executor) | +| runner.kubernetesAnnotations | list | `[]` | Set additional annotations on the pods created for kubernetes jobs. Name and value are separated by an equals sign, e.g. `cluster-autoscaler.kubernetes.io/safe-to-evict=false`. | | runner.kubernetesJobImage | string | `""` | Set the kubernetes job image. Defaults to `leg100/otf-job` tagged with the current OTF version. See [docs](https://docs.otf.ninja/config/flags/#-kubernetes-job-image). | | runner.kubernetesLabels | list | `[]` | Set additional labels on kubernetes jobs. Name and value are separated by an equals sign, e.g. `foo=bar`. | | runner.kubernetesLimitCPU | string | `nil` | Set a CPU limit for kubernetes jobs. | diff --git a/charts/otfd/Chart.yaml b/charts/otfd/Chart.yaml index 59308975b..51c26c393 100644 --- a/charts/otfd/Chart.yaml +++ b/charts/otfd/Chart.yaml @@ -13,7 +13,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.2 +version: 0.4.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. diff --git a/charts/otfd/README.md b/charts/otfd/README.md index f53e6b623..582eeff27 100644 --- a/charts/otfd/README.md +++ b/charts/otfd/README.md @@ -1,6 +1,6 @@ # Helm chart for `otfd` -![Version: 0.4.2](https://img.shields.io/badge/Version-0.4.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.3](https://img.shields.io/badge/AppVersion-0.6.3-informational?style=flat-square) +![Version: 0.4.3](https://img.shields.io/badge/Version-0.4.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.3](https://img.shields.io/badge/AppVersion-0.6.3-informational?style=flat-square) Installs the [otf](https://github.com/leg100/otf) daemon. @@ -111,6 +111,7 @@ Note: you should only use this for testing purposes. | runner.cacheVolume.storageClass | string | `nil` | Persistent volume storage class. # If defined, storageClassName: # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is # set, choosing the default provisioner. | | runner.concurrency | int | `nil` | Set the number of runs that can be processed concurrently. See [docs](https://docs.otf.ninja/config/flags/#-concurrency). | | runner.executor | string | `""` | The executor to use. See [docs](https://docs.otf.ninja/config/flags/#-executor) | +| runner.kubernetesAnnotations | list | `[]` | Set additional annotations on the pods created for kubernetes jobs. Name and value are separated by an equals sign, e.g. `cluster-autoscaler.kubernetes.io/safe-to-evict=false`. | | runner.kubernetesJobImage | string | `""` | Set the kubernetes job image. Defaults to `leg100/otf-job` tagged with the current OTF version. See [docs](https://docs.otf.ninja/config/flags/#-kubernetes-job-image). | | runner.kubernetesLabels | list | `[]` | Set additional labels on kubernetes jobs. Name and value are separated by an equals sign, e.g. `foo=bar`. | | runner.kubernetesLimitCPU | string | `nil` | Set a CPU limit for kubernetes jobs. | diff --git a/docs/docs/config/flags.md b/docs/docs/config/flags.md index 6858c366e..ae00f2cc2 100644 --- a/docs/docs/config/flags.md +++ b/docs/docs/config/flags.md @@ -170,6 +170,13 @@ Set a memory limit for kubernetes jobs. Set additional labels on kubernetes jobs. Name and value are separated by an equals sign, e.g. `foo=bar`. +## `--kubernetes-annotations` + +* System: `otfd`, `otf-agent` +* Default: `""` + +Set additional annotations on the pods created for kubernetes jobs. Name and value are separated by an equals sign, e.g. `cluster-autoscaler.kubernetes.io/safe-to-evict=false`. Specify multiple annotations separated by a comma. Use this to set the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-types-of-pods-can-prevent-ca-from-removing-a-node) `cluster-autoscaler.kubernetes.io/safe-to-evict` annotation so that in-progress runs are not disrupted by node scale-down. + ## `--kubernetes-ttl-after-finish` * System: `otfd`, `otf-agent` diff --git a/docs/docs/executors.md b/docs/docs/executors.md index 8336baa0b..d54c13a49 100644 --- a/docs/docs/executors.md +++ b/docs/docs/executors.md @@ -26,6 +26,7 @@ This executor is only functional when `otfd` or `otf-agent` is deployed via the There are a number of flags that customise the jobs: * [`--kubernetes-job-image`](config/flags.md#-kubernetes-job-image) +* [`--kubernetes-annotations`](config/flags.md#-kubernetes-annotations) * [`--kubernetes-request-cpu`](config/flags.md#-kubernetes-request-cpu) * [`--kubernetes-request-memory`](config/flags.md#-kubernetes-request-memory) * [`--kubernetes-ttl-after-finish`](config/flags.md#-kubernetes-ttl-after-finish) diff --git a/internal/runner/executor_kube.go b/internal/runner/executor_kube.go index 44a021362..8d425adbc 100644 --- a/internal/runner/executor_kube.go +++ b/internal/runner/executor_kube.go @@ -87,6 +87,7 @@ type kubeConfig struct { limitCPU *k8sresource.Quantity limitMemory *k8sresource.Quantity labels map[string]string + annotations map[string]string flags kubeConfigFlags } @@ -95,6 +96,7 @@ type kubeConfig struct { // used directly by the kubernetes executor. type kubeConfigFlags struct { Labels []string + Annotations []string RequestCPU string RequestMemory string LimitCPU string @@ -109,6 +111,7 @@ func registerKubeFlags(flags *pflag.FlagSet, cfg *kubeConfig) { flags.StringVar(&cfg.flags.LimitCPU, "kubernetes-limit-cpu", cfg.flags.LimitCPU, "CPU limit for kubernetes job.") flags.StringVar(&cfg.flags.LimitMemory, "kubernetes-limit-memory", cfg.flags.LimitMemory, "Memory limit for kubernetes job.") flags.StringSliceVar(&cfg.flags.Labels, "kubernetes-labels", cfg.flags.Labels, "Set additional labels on kubernetes jobs. Name and value are separated by an equals sign, e.g. `foo=bar`.") + flags.StringSliceVar(&cfg.flags.Annotations, "kubernetes-annotations", cfg.flags.Annotations, "Set additional annotations on the pods created for kubernetes jobs. Name and value are separated by an equals sign, e.g. `cluster-autoscaler.kubernetes.io/safe-to-evict=false`.") } type kubeExecutor struct { @@ -177,6 +180,15 @@ func newKubeExecutor( executor.Config.labels[k] = v } + executor.Config.annotations = make(map[string]string) + for _, annotation := range kubeConfig.flags.Annotations { + k, v, ok := strings.Cut(annotation, "=") + if !ok { + return nil, fmt.Errorf("invalid annotation: must be in format name=value") + } + executor.Config.annotations[k] = v + } + // assume running in-cluster; otherwise use config path config, err := rest.InClusterConfig() if errors.Is(err, rest.ErrNotInCluster) { @@ -210,6 +222,9 @@ func (s *kubeExecutor) SpawnOperation(ctx context.Context, _ *errgroup.Group, jo } maps.Copy(labels, s.Config.labels) + annotations := make(map[string]string, len(s.Config.annotations)) + maps.Copy(annotations, s.Config.annotations) + const ( cacheVolumeName = "cache" jobTokenSecretKey = "jobToken" @@ -260,7 +275,8 @@ func (s *kubeExecutor) SpawnOperation(ctx context.Context, _ *errgroup.Group, jo TTLSecondsAfterFinished: new(int32(s.Config.TTLAfterFinish.Seconds())), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: labels, + Labels: labels, + Annotations: annotations, }, Spec: corev1.PodSpec{ ServiceAccountName: s.Config.ServiceAccount, diff --git a/internal/runner/executor_kube_test.go b/internal/runner/executor_kube_test.go index 0b39de315..7af057b01 100644 --- a/internal/runner/executor_kube_test.go +++ b/internal/runner/executor_kube_test.go @@ -74,11 +74,40 @@ func TestNewKubeExecutor(t *testing.T) { ) assert.Error(t, err) }) + + t.Run("with annotations", func(t *testing.T) { + cfg := defaultKubeConfig + cfg.flags.Annotations = []string{"cluster-autoscaler.kubernetes.io/safe-to-evict=false", "coo=boo"} + + executor, err := newKubeExecutor( + logr.Discard(), + defaultOperationConfig(), + cfg, + ) + require.NoError(t, err) + assert.Equal(t, map[string]string{ + "cluster-autoscaler.kubernetes.io/safe-to-evict": "false", + "coo": "boo", + }, executor.Config.annotations) + }) + + t.Run("with invalid annotations", func(t *testing.T) { + cfg := defaultKubeConfig + cfg.flags.Annotations = []string{"foobar", "cooboo"} + + _, err := newKubeExecutor( + logr.Discard(), + defaultOperationConfig(), + cfg, + ) + assert.Error(t, err) + }) } func TestKubeExecutor_SpawnOperation(t *testing.T) { cfg := defaultKubeConfig cfg.flags.Labels = []string{"foo=bar"} + cfg.flags.Annotations = []string{"cluster-autoscaler.kubernetes.io/safe-to-evict=false"} cfg.flags.LimitCPU = "3000m" cfg.flags.LimitMemory = "512Mi" @@ -123,6 +152,13 @@ func TestKubeExecutor_SpawnOperation(t *testing.T) { assert.Equal(t, wantLabels, jobsClient.job.Labels) assert.Equal(t, wantLabels, secretsClient.secret.Labels) assert.Equal(t, map[string]string{"jobToken": "token"}, secretsClient.secret.StringData) + + // Annotations are set on the pods (via the pod template), not on the job + // or secret. + assert.Equal(t, + map[string]string{"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, + jobsClient.job.Spec.Template.Annotations, + ) } type fakeSecretsClient struct {