diff --git a/src/compute-plane-services/nvca/internal/miniservice/status.go b/src/compute-plane-services/nvca/internal/miniservice/status.go index b5bc61f97e..b392898c80 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/status.go +++ b/src/compute-plane-services/nvca/internal/miniservice/status.go @@ -1374,6 +1374,8 @@ func (r *Reconciler) getUnexpectedEventsForObject( // parseErrorEventMessage parses event.Message to make it more human-readable and remove // internal cluster details when possible. It returns false when event is not an error message. func parseErrorEventMessage(event *corev1.Event) (message string, include, isError bool) { + const emStrForbidden = "forbidden:" + switch event.Reason { case "FailedCreate", "FailedUpdate": // FailedCreate/Update is added to an event and/or in a ReplicaSet or StatefulSet condition @@ -1390,7 +1392,23 @@ func parseErrorEventMessage(event *corev1.Event) (message string, include, isErr // https://github.com/kubernetes/kubernetes/blob/25f1248/pkg/controller/controller_utils.go#L596C75-L596C89 // https://github.com/kubernetes/kubernetes/blob/25f1248/pkg/controller/statefulset/stateful_pod_control.go#L297-L300 // https://github.com/kubernetes/kubernetes/blob/25f1248/pkg/controller/statefulset/stateful_pod_control.go#L314-L317 + // + // A transient webhook-unavailability signature is retryable, not + // terminal: marking it terminal force-purges the object and + // regenerates the load that caused the failure. + if isTransientWebhookUnavailableMessage(event.Message) { + return event.Message, event.Type == corev1.EventTypeWarning, false + } isError = true + case "BindingError": + // Same transient-webhook carve-out as FailedCreate/FailedUpdate; + // BindingError can carry the same webhook-unavailable signature. + if isTransientWebhookUnavailableMessage(event.Message) { + return event.Message, event.Type == corev1.EventTypeWarning, false + } + // Non-webhook messages (e.g. a routine resourceVersion conflict) + // fall back to the same forbidden:-only rule as default. + isError = strings.Contains(event.Message, emStrForbidden) case "ReplicaSetCreateError": // ReplicaSetCreateError is set when Deployments fail to create their ReplicaSet's. // @@ -1406,13 +1424,51 @@ func parseErrorEventMessage(event *corev1.Event) (message string, include, isErr default: // Controllers for objects that create Pods may observe "forbidden" errors, // but these do not necessarily fail the controller object. - const emStrForbidden = "forbidden:" isError = isError || strings.Contains(event.Message, emStrForbidden) } return event.Message, isError || event.Type == corev1.EventTypeWarning, isError } +// Webhook-unavailability message substrings. +var transientWebhookUnavailableSignatures = []string{ + // Webhook Service has no ready endpoints, e.g. during a rollout or after node loss. + "no endpoints available for service", + + // Webhook returned 5xx or the API server could not read a response. + "the server is currently unable to handle the request", + "has prevented the request from succeeding", + "too many requests", + + // Connection could not be established or was lost mid-request. + "connection refused", + "connection reset by peer", + "broken pipe", + "http2: client connection lost", + "EOF", + + // Webhook did not respond in time. + "i/o timeout", + "context deadline exceeded", + "net/http: TLS handshake timeout", + "net/http: request canceled", +} + +// isTransientWebhookUnavailableMessage reports whether an event message +// matches a known webhook-unavailable signature, not a permanent failure. +func isTransientWebhookUnavailableMessage(message string) bool { + lowerMessage := strings.ToLower(message) + if !strings.Contains(lowerMessage, "failed calling webhook") { + return false + } + for _, signature := range transientWebhookUnavailableSignatures { + if strings.Contains(lowerMessage, strings.ToLower(signature)) { + return true + } + } + return false +} + var ( filterTerminal = func(os ObjectStatus) bool { return os.TerminalBad } filterPending = func(os ObjectStatus) bool { return os.Pending } diff --git a/src/compute-plane-services/nvca/internal/miniservice/status_test.go b/src/compute-plane-services/nvca/internal/miniservice/status_test.go index 49cf59c737..a024b71ca5 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/status_test.go +++ b/src/compute-plane-services/nvca/internal/miniservice/status_test.go @@ -159,6 +159,68 @@ func Test_parseErrorEventMessage(t *testing.T) { expInclude: true, expIsError: false, }, + { + name: "transient FailedCreate - webhook connection reset (hijacked/reset, not a timeout)", + event: corev1.Event{ + Type: corev1.EventTypeWarning, + Reason: "FailedCreate", + Message: `create Pod nvcf-test-func-0 in StatefulSet nvcf-test-func failed error: ` + + `Internal error occurred: failed calling webhook "mutate-pod-nodeaffinity.nvca.nvcf.nvidia.io": ` + + `failed to call webhook: an error on the server ("") has prevented the request from succeeding`, + }, + expInclude: true, + expIsError: false, + }, + { + name: "transient BindingError - webhook EOF", + event: corev1.Event{ + Type: corev1.EventTypeWarning, + Reason: "BindingError", + Message: `Post "https://nvca.nvca-system.svc:8443/validate": ` + + `failed calling webhook "validate-helm-charts.nvca.nvcf.nvidia.io": ` + + `Post "https://nvca.nvca-system.svc:8443/validate": EOF`, + }, + expInclude: true, + expIsError: false, + }, + { + name: "transient FailedCreate - webhook signature match is case-insensitive", + event: corev1.Event{ + Type: corev1.EventTypeWarning, + Reason: "FailedCreate", + Message: `create Pod nvcf-test-func-0 in StatefulSet nvcf-test-func failed error: ` + + `Internal error occurred: Failed Calling Webhook "mutate-pod-nodeaffinity.nvca.nvcf.nvidia.io": ` + + `failed to call webhook: an error on the server ("") has prevented the Request From Succeeding`, + }, + expInclude: true, + expIsError: false, + }, + { + name: "non-transient BindingError should still be an error", + event: corev1.Event{ + Type: corev1.EventTypeWarning, + Reason: "BindingError", + // A real denial (matches the forbidden: rule every other + // Reason's non-transient path already relies on), not a + // resourceVersion conflict -- conflicts are routine, + // self-healing, and auto-retried by controllers, so using + // one here would misrepresent what a genuine terminal + // BindingError looks like. + Message: `pods "foo-0" is forbidden: ` + exceededQuotaMsg, + }, + expInclude: true, + expIsError: true, + }, + { + name: "transient BindingError - resourceVersion conflict is routine, not terminal", + event: corev1.Event{ + Type: corev1.EventTypeWarning, + Reason: "BindingError", + Message: `Operation cannot be fulfilled on pods "foo-0": the object has been modified`, + }, + expInclude: true, + expIsError: false, + }, { name: "policy violation warning should be excluded", event: corev1.Event{ @@ -178,6 +240,24 @@ func Test_parseErrorEventMessage(t *testing.T) { } } +func Test_isTransientWebhookUnavailableMessage(t *testing.T) { + const prefix = `Internal error occurred: failed calling webhook "mutate-pod-nodeaffinity.nvca.nvcf.nvidia.io": ` + + for _, signature := range transientWebhookUnavailableSignatures { + t.Run(signature, func(t *testing.T) { + assert.True(t, isTransientWebhookUnavailableMessage(prefix+signature)) + }) + } + + t.Run("no signature match", func(t *testing.T) { + assert.False(t, isTransientWebhookUnavailableMessage(prefix+"x509: certificate signed by unknown authority")) + }) + + t.Run("signature present but not a webhook call failure", func(t *testing.T) { + assert.False(t, isTransientWebhookUnavailableMessage("connection refused")) + }) +} + func Test_ObjectStatuses_backoffBehavior(t *testing.T) { now := time.Now() testCfg := testTimeConfig()