From 9f84d581e9c96fbdf60b09302f8e618c0d103a34 Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Fri, 4 Sep 2026 10:48:34 -0700 Subject: [PATCH 1/2] fix(nvca): gate NVLink ComputeDomain allocation on domain-index annotation NVCA's mutating webhook attached an IMEX ComputeDomain channel resource claim to every GPU-requesting Pod on an NVLink-optimized cluster, regardless of whether the Pod actually needed cross-node NVLink memory sharing. Since each node exposes only one DRA channel device, this made any claiming Pod the exclusive GPU tenant of its node, capping bin-packing at one GPU Pod per node cluster-wide. Gate ComputeDomain and channel claim attachment on the existing dra.nvcf.nvidia.io/required-nvlink-domain-index annotation's presence: a Pod that never set it was never depending on ComputeDomain-backed placement guarantees, so removing its claim is not a breaking change. Also fix a corollary issue: NVCA created a single shared ComputeDomain for a whole function regardless of how many distinct domain-index values were present, when a ComputeDomain represents one IMEX domain and each distinct index is meant to be an independent NVLink domain. pkg/dra now creates one ComputeDomain per distinct index and the reconciler passes the resulting raw-value-to-ComputeDomain mapping to the webhook through the existing miniservice metadata ConfigMap, since index normalization must happen once, across the whole set of a function's rendered objects. While wiring this up, also fix an annotation-location bug in the existing domain-index grouping logic: it read annotations off the top-level controller object (Deployment/StatefulSet/etc.) instead of its Pod template, which is the only location Kubernetes copies onto the Pods the webhook admits. Update docs/user/helm-functions.md and docs/user/cluster-management/topology-aware-scheduling.md: the required-nvlink-domain-index annotation was documented as optional legacy scheduling guidance; it is now also the required signal for ComputeDomain allocation. Signed-off-by: Eric Stroczynski --- .../topology-aware-scheduling.md | 8 +- docs/user/helm-functions.md | 42 ++- .../templates/statefulset.yaml | 9 +- .../miniservice/metadata_configmap.go | 3 + .../nvca/internal/miniservice/reconcile.go | 9 +- .../internal/miniservice/reconcile_test.go | 58 +++- .../nvca/pkg/dra/dra.go | 173 +++++++++--- .../nvca/pkg/dra/dra_test.go | 258 ++++++++++++------ .../nvca/pkg/types/miniservice_types.go | 8 + .../webhook/miniservice_mutating_webhook.go | 25 +- .../miniservice_mutating_webhook_test.go | 62 +++++ 11 files changed, 494 insertions(+), 161 deletions(-) diff --git a/docs/user/cluster-management/topology-aware-scheduling.md b/docs/user/cluster-management/topology-aware-scheduling.md index 12fc5adbfc..868ff4bee6 100644 --- a/docs/user/cluster-management/topology-aware-scheduling.md +++ b/docs/user/cluster-management/topology-aware-scheduling.md @@ -22,11 +22,13 @@ See the and [Dynamo topology guide](https://docs.nvidia.com/dynamo/v1.4.1/kubernetes-deployment/scale/topology-aware-scheduling) for component-level details. -Helm functions can also use the legacy -[`dra.nvcf.nvidia.io` partition annotation](../helm-functions.md#legacy-nvca-nvlink-partition-annotation). +Helm functions can also use the +[`dra.nvcf.nvidia.io` partition annotation](../helm-functions.md#required-nvlink-domain-index-annotation). That path uses Kubernetes Pod affinity and is best-effort without KAI Scheduler or Grove topology-aware scheduling. Use the KAI or Grove mechanisms on this -page when clique placement must be coordinated for the complete workload. +page when clique placement must be coordinated for the complete workload. The +annotation is still required for NVCA to allocate a `ComputeDomain` for a +function's Pods, even when using KAI or Grove for placement. ## Prerequisites diff --git a/docs/user/helm-functions.md b/docs/user/helm-functions.md index 74098c93c7..14eaa61442 100644 --- a/docs/user/helm-functions.md +++ b/docs/user/helm-functions.md @@ -98,10 +98,10 @@ to connect GPU workload Pods through IMEX. Each GPU-enabled Pod must request a full node of GPUs. -#### Legacy NVCA NVLink partition annotation +#### Required NVLink domain index annotation -NVLink partition placement through these legacy affinity rules is best-effort +NVLink partition placement through these affinity rules is best-effort without KAI Scheduler or Grove topology-aware scheduling. The rules do not provide atomic gang placement. Concurrent Pods can initially land in different cliques, and distinct logical groups can land in the same clique. Use @@ -111,8 +111,16 @@ when the workload requires all Pods to fit and start in a specific topology domain. + +This annotation is not optional legacy configuration. It is the signal NVCA +uses to decide whether a Pod needs a `ComputeDomain` and IMEX channel claim +at all. Set it even for charts that use KAI Scheduler or Grove topology +constraints for scheduling, if the function needs cross-node NVLink memory +sharing. + + For charts that do not use KAI Scheduler or Grove topology constraints, NVCA -supports this legacy Pod template annotation: +supports this Pod template annotation: ```yaml spec: @@ -136,6 +144,27 @@ rack or clique: require the groups to use different GPU cliques. - Pods without the annotation share a default logical group. +##### ComputeDomain allocation + +NVCA uses this annotation's presence, not just its value, as the signal for +whether a Pod needs a `ComputeDomain` and IMEX channel resource claim at all: + +- A Pod with the annotation set has declared that it needs to be scheduled + into a specific NVLink domain, which only makes sense if it does cross-node + NVLink memory sharing. NVCA allocates a `ComputeDomain` for that domain and + attaches a channel resource claim to the Pod's GPU containers. +- A Pod without the annotation has no declared NVLink domain requirement. Its + GPU peers can already land in different, unrelated NVLink domains, or on + nodes with no NVLink domain membership at all. NVCA does not attach a + `ComputeDomain` claim to it, since the claim would provide no functional + benefit to a Pod that never declared a cross-node NVLink requirement. + +NVCA creates one `ComputeDomain` per distinct annotation value present in the +function, not one shared `ComputeDomain` for the whole function. A +`ComputeDomain` represents a single IMEX domain, so Pods that set different +values are meant to join different, independent NVLink domains and each gets +its own `ComputeDomain` and channel. + On an NVLink-optimized cluster, NVCA mutates each admitted Pod as follows: - It adds the generated `dra.nvcf.nvidia.io/nvlink-domain-partition` label. @@ -145,9 +174,10 @@ On an NVLink-optimized cluster, NVCA mutates each admitted Pod as follows: - For an unannotated Pod, it adds preferred Pod affinity with weight 100. The scheduler can spread these Pods when it cannot satisfy the preference. - It requires placement on a node that has the `nvidia.com/gpu.clique` label. -- It adds the function's `ComputeDomain` resource claim to containers that - request `nvidia.com/gpu`, `nvidia.com/pgpu`, `nvidia.com/gpu.shared`, or an - `nvidia.com/mig-*` resource. +- For an annotated Pod, it adds the resource claim for the `ComputeDomain` + provisioned for that Pod's annotation value to containers that request + `nvidia.com/gpu`, `nvidia.com/pgpu`, `nvidia.com/gpu.shared`, or an + `nvidia.com/mig-*` resource. An unannotated Pod never receives this claim. ## Limitations diff --git a/examples/function-samples/helmchart-samples/multi-node-helm-function-test/multi-node-test/templates/statefulset.yaml b/examples/function-samples/helmchart-samples/multi-node-helm-function-test/multi-node-test/templates/statefulset.yaml index ea648756e5..02bb7ea02b 100644 --- a/examples/function-samples/helmchart-samples/multi-node-helm-function-test/multi-node-test/templates/statefulset.yaml +++ b/examples/function-samples/helmchart-samples/multi-node-helm-function-test/multi-node-test/templates/statefulset.yaml @@ -45,12 +45,13 @@ spec: metadata: {{- with $profile.podAnnotations }} annotations: - # Legacy NVCA placement on an NVLink-optimized cluster: + # NVCA placement and ComputeDomain allocation on an NVLink-optimized cluster: # - Set dra.nvcf.nvidia.io/required-nvlink-domain-index to a logical group index, - # such as "0", for required pod affinity among Pods with the same value. + # such as "0", for required pod affinity among Pods with the same value, and to + # have NVCA allocate a ComputeDomain and IMEX channel claim for this Pod. # - Omit dra.nvcf.nvidia.io/required-nvlink-domain-index for preferred - # same-clique placement. - # These options are best-effort without KAI Scheduler or Grove + # same-clique placement with no ComputeDomain or channel claim. + # Placement through these options is best-effort without KAI Scheduler or Grove # topology-aware scheduling. Different indices do not guarantee # placement in different physical NVLink partitions. {{- toYaml . | nindent 8 }} diff --git a/src/compute-plane-services/nvca/internal/miniservice/metadata_configmap.go b/src/compute-plane-services/nvca/internal/miniservice/metadata_configmap.go index 22a087176e..1f0b421067 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/metadata_configmap.go +++ b/src/compute-plane-services/nvca/internal/miniservice/metadata_configmap.go @@ -30,6 +30,7 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v1alpha1" nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvcfdra "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/dra" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/featureflag" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nodefeatures" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/enforce/kaischeduler" @@ -50,6 +51,7 @@ type MetadataInput struct { OTelCollectorEnvVars []corev1.EnvVar TerminationGracePeriodSeconds *int64 ModelCacheInitEnv map[string]string + NVLinkComputeDomains map[string]nvcfdra.ComputeDomainRef } // buildMiniserviceMetadata constructs a MiniserviceMetadata from the controller's @@ -86,6 +88,7 @@ func (r *Reconciler) buildMiniserviceMetadata( ImagePullSecretNames: secretNames, TerminationGracePeriodSeconds: in.TerminationGracePeriodSeconds, ModelCacheInitEnv: in.ModelCacheInitEnv, + NVLinkComputeDomains: in.NVLinkComputeDomains, } meta.Labels, meta.Annotations = newGeneralObjectLabelsAndAnnotations( diff --git a/src/compute-plane-services/nvca/internal/miniservice/reconcile.go b/src/compute-plane-services/nvca/internal/miniservice/reconcile.go index 8da0b3de5b..e5b3bf222c 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/reconcile.go +++ b/src/compute-plane-services/nvca/internal/miniservice/reconcile.go @@ -846,7 +846,14 @@ func (r *Reconciler) doInstall(ctx context.Context, infraObjs = append(infraObjs, utilsPod) if r.FeatureFlagFetcher.IsAttributeEnabled(featureflag.AttrNVLinkOptimized) { - infraObjs = append(infraObjs, nvcfdra.NewSingleChannelComputeDomain()) + cds, refs, err := nvcfdra.ComputeDomainsForWorkload(workloadObjs...) + if err != nil { + return reconcile.Result{}, reconcile.TerminalError(fmt.Errorf("compute NVLink ComputeDomains: %w", err)) + } + for _, cd := range cds { + infraObjs = append(infraObjs, cd) + } + metaInput.NVLinkComputeDomains = refs } // Create the miniservice metadata ConfigMap before any objects are created diff --git a/src/compute-plane-services/nvca/internal/miniservice/reconcile_test.go b/src/compute-plane-services/nvca/internal/miniservice/reconcile_test.go index 4b9a99ad97..e2ee3a4426 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/reconcile_test.go +++ b/src/compute-plane-services/nvca/internal/miniservice/reconcile_test.go @@ -1682,7 +1682,7 @@ func TestReconcile_Function_NVLinkOptimized(t *testing.T) { }, }, } - testReconcileNVLinkOptimizedHelper(t, helmObjs, expDeployments) + testReconcileNVLinkOptimizedHelper(t, helmObjs, expDeployments, nil) }) t.Run("required domains", func(t *testing.T) { helmObjs := []client.Object{ @@ -1693,12 +1693,14 @@ func TestReconcile_Function_NVLinkOptimized(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "foo1", - Annotations: map[string]string{ - nvcfdra.RequiredNVLinkDomainIndexAnnotation: "0", - }, }, Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + nvcfdra.RequiredNVLinkDomainIndexAnnotation: "0", + }, + }, Spec: corev1.PodSpec{ Containers: []corev1.Container{{ Name: "test", @@ -1719,12 +1721,14 @@ func TestReconcile_Function_NVLinkOptimized(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "foo2", - Annotations: map[string]string{ - nvcfdra.RequiredNVLinkDomainIndexAnnotation: "1", - }, }, Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + nvcfdra.RequiredNVLinkDomainIndexAnnotation: "1", + }, + }, Spec: corev1.PodSpec{ Containers: []corev1.Container{{ Name: "test", @@ -1741,16 +1745,20 @@ func TestReconcile_Function_NVLinkOptimized(t *testing.T) { } // Workload objects are bare Helm renders; DRA resource claims, NVLink labels, // affinity, and service account are injected by the webhook at Pod admission. + // The required-nvlink-domain-index annotation lives on the pod template, since that + // is the only location Kubernetes copies down onto the Pods the webhook admits. expDeployments := []appsv1.Deployment{ { ObjectMeta: metav1.ObjectMeta{ Name: "foo1", - Annotations: map[string]string{ - nvcfdra.RequiredNVLinkDomainIndexAnnotation: "0", - }, }, Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + nvcfdra.RequiredNVLinkDomainIndexAnnotation: "0", + }, + }, Spec: corev1.PodSpec{ Containers: []corev1.Container{{ Name: "test", @@ -1768,12 +1776,14 @@ func TestReconcile_Function_NVLinkOptimized(t *testing.T) { { ObjectMeta: metav1.ObjectMeta{ Name: "foo2", - Annotations: map[string]string{ - nvcfdra.RequiredNVLinkDomainIndexAnnotation: "1", - }, }, Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + nvcfdra.RequiredNVLinkDomainIndexAnnotation: "1", + }, + }, Spec: corev1.PodSpec{ Containers: []corev1.Container{{ Name: "test", @@ -1789,11 +1799,16 @@ func TestReconcile_Function_NVLinkOptimized(t *testing.T) { }, }, } - testReconcileNVLinkOptimizedHelper(t, helmObjs, expDeployments) + testReconcileNVLinkOptimizedHelper(t, helmObjs, expDeployments, []string{"nvcf-cd-index-1", "nvcf-cd-index-2"}) }) } -func testReconcileNVLinkOptimizedHelper(t *testing.T, helmObjs []client.Object, expDeployments []appsv1.Deployment) { +func testReconcileNVLinkOptimizedHelper( + t *testing.T, + helmObjs []client.Object, + expDeployments []appsv1.Deployment, + expComputeDomainNames []string, +) { ctx := newTestContext() testScheme := mgrScheme @@ -2154,6 +2169,19 @@ rules: assert.Equal(t, expDeployment, gotDeployment) } + // One ComputeDomain must be created per distinct required-nvlink-domain-index value present + // in the workload objects, and none at all when nothing declares one, since a ComputeDomain + // represents a single IMEX domain. + gotComputeDomains := &nvresourcev1beta1.ComputeDomainList{} + err = r.Client.List(ctx, gotComputeDomains, client.InNamespace(ms.Spec.Namespace)) + require.NoError(t, err) + gotComputeDomainNames := make([]string, len(gotComputeDomains.Items)) + for i, cd := range gotComputeDomains.Items { + gotComputeDomainNames[i] = cd.Name + } + sort.Strings(gotComputeDomainNames) + assert.ElementsMatch(t, expComputeDomainNames, gotComputeDomainNames) + err = r.Client.Get(ctx, client.ObjectKeyFromObject(ms), ms) require.NoError(t, err) assert.Equal(t, v1alpha1.MiniServiceInstalled, ms.Status.Phase) diff --git a/src/compute-plane-services/nvca/pkg/dra/dra.go b/src/compute-plane-services/nvca/pkg/dra/dra.go index 513e07da20..e16b3b1c77 100644 --- a/src/compute-plane-services/nvca/pkg/dra/dra.go +++ b/src/compute-plane-services/nvca/pkg/dra/dra.go @@ -60,28 +60,38 @@ func TransformNVLinkOptimizedDRAObjects( prefNVDObjs := []client.Object{} objsByReqNVDIndex := map[int][]client.Object{} for _, sourceObj := range sourceObjs { - annos := sourceObj.GetAnnotations() - if annos == nil { + idxStr, ok := podTemplateAnnotation(sourceObj, RequiredNVLinkDomainIndexAnnotation) + if !ok { prefNVDObjs = append(prefNVDObjs, sourceObj) continue } - if idxStr, ok := annos[RequiredNVLinkDomainIndexAnnotation]; ok { - idx := reqNVDIndexMap[idxStr] - objsByReqNVDIndex[idx] = append(objsByReqNVDIndex[idx], sourceObj) - } else { - prefNVDObjs = append(prefNVDObjs, sourceObj) - } + idx := reqNVDIndexMap[idxStr] + objsByReqNVDIndex[idx] = append(objsByReqNVDIndex[idx], sourceObj) } - cd := NewSingleChannelComputeDomain() - SetComputeDomainToGPUPodResourceClaims(cd, sourceObjs...) - SetPreferredNVLinkDomainSchedulingParameters(keyToHash, prefNVDObjs...) - for idx, objs := range objsByReqNVDIndex { + + // Objects with no required-domain-index annotation never join a ComputeDomain: they were + // never depending on ComputeDomain-backed placement guarantees, so no claim is attached for + // them. Each distinct required index gets its own ComputeDomain, since a ComputeDomain + // represents one IMEX domain and objects in different index groups are meant to join + // different, independent NVLink domains. + indices := make([]int, 0, len(objsByReqNVDIndex)) + for idx := range objsByReqNVDIndex { + indices = append(indices, idx) + } + sort.Ints(indices) + + cds := make([]client.Object, 0, len(indices)) + for _, idx := range indices { + objs := objsByReqNVDIndex[idx] + cd := computeDomainForIndex(idx) + SetComputeDomainToGPUPodResourceClaims(cd, objs...) SetRequiredNVLinkDomainSchedulingParameters(keyToHash, fmt.Sprint(idx), objs...) + cds = append(cds, cd) } - return sourceObjs, []client.Object{cd}, nil + return sourceObjs, cds, nil } func sanitizeIndices(objs []client.Object) (map[string]int, error) { @@ -94,24 +104,22 @@ func sanitizeIndices(objs []client.Object) (map[string]int, error) { var indexTuples []strIndexTuple indexSet := sets.New[string]() for _, sourceObj := range objs { - annos := sourceObj.GetAnnotations() - if annos == nil { + idx, ok := podTemplateAnnotation(sourceObj, RequiredNVLinkDomainIndexAnnotation) + if !ok { continue } - if idx, ok := annos[RequiredNVLinkDomainIndexAnnotation]; ok { - if indexSet.Has(idx) { - continue - } - indexSet.Insert(idx) - i, err := strconv.ParseInt(idx, 10, 32) - if err != nil { - return nil, err - } - indexTuples = append(indexTuples, strIndexTuple{ - i: int(i), - s: idx, - }) + if indexSet.Has(idx) { + continue + } + indexSet.Insert(idx) + i, err := strconv.ParseInt(idx, 10, 32) + if err != nil { + return nil, err } + indexTuples = append(indexTuples, strIndexTuple{ + i: int(i), + s: idx, + }) } if len(indexTuples) == 0 { indexTuples = append(indexTuples, strIndexTuple{ @@ -134,24 +142,113 @@ func sanitizeIndices(objs []client.Object) (map[string]int, error) { } const ( - defaultComputeDomainName = "nvcf-cd-index-0" - defaultComputeDomainChannelName = "nvcf-cd-channel-0" + computeDomainNamePrefix = "nvcf-cd-index" + computeDomainChannelNamePrefix = "nvcf-cd-channel" ) -func NewSingleChannelComputeDomain() *nvresourcev1beta1.ComputeDomain { - cd := &nvresourcev1beta1.ComputeDomain{ +// ComputeDomainRef identifies a ComputeDomain and the name of the ResourceClaimTemplate that +// backs its channel, without requiring callers to hold the full ComputeDomain object. It is the +// shape persisted into the MiniserviceMetadata ConfigMap so the admission webhook can attach +// claims to a pod without recomputing or renaming the ComputeDomain itself. +type ComputeDomainRef struct { + ComputeDomainName string `json:"computeDomainName"` + ChannelName string `json:"channelName"` +} + +// ComputeDomainFromRef builds the ComputeDomain object identified by ref. Used by callers (the +// admission webhook) that only have a ComputeDomainRef, not the originating object set. +func ComputeDomainFromRef(ref ComputeDomainRef) *nvresourcev1beta1.ComputeDomain { + return &nvresourcev1beta1.ComputeDomain{ ObjectMeta: metav1.ObjectMeta{ - Name: defaultComputeDomainName, + Name: ref.ComputeDomainName, }, Spec: nvresourcev1beta1.ComputeDomainSpec{ Channel: &nvresourcev1beta1.ComputeDomainChannelSpec{ ResourceClaimTemplate: nvresourcev1beta1.ComputeDomainResourceClaimTemplate{ - Name: defaultComputeDomainChannelName, + Name: ref.ChannelName, }, }, }, } - return cd +} + +// computeDomainForIndex builds the ComputeDomain for a single normalized NVLink domain index. +// Naming is index-suffixed so that objects declaring different required-nvlink-domain-index +// values are backed by distinct ComputeDomains, matching the one-IMEX-domain-per-ComputeDomain +// invariant: a ComputeDomain represents one IMEX domain, so objects meant to join different, +// independent NVLink domains must not share one. +func computeDomainForIndex(idx int) *nvresourcev1beta1.ComputeDomain { + return ComputeDomainFromRef(ComputeDomainRef{ + ComputeDomainName: fmt.Sprintf("%s-%d", computeDomainNamePrefix, idx), + ChannelName: fmt.Sprintf("%s-%d", computeDomainChannelNamePrefix, idx), + }) +} + +// podTemplateAnnotation returns the value of annotation key on obj's pod template (or on obj +// itself when obj is a bare Pod, since a Pod is its own template), and whether it was present. +// This is the same location Kubernetes copies onto the Pods a Deployment/StatefulSet/Job/CronJob +// creates, so it is the only place an annotation set here is guaranteed to reach the admission +// webhook, which only ever sees realized Pods. A top-level annotation on the controller object +// itself (e.g. a Deployment's own ObjectMeta) is never copied down and would never reach a Pod. +func podTemplateAnnotation(obj client.Object, key string) (string, bool) { + var val string + var ok bool + itrf := func(pts *corev1.PodTemplateSpec) { + if v, found := pts.Annotations[key]; found { + val, ok = v, true + } + } + iterPodSpecs(itrf, obj) + return val, ok +} + +// anyPodHasRequiredNVLinkDomainIndex reports whether any pod template among objs carries the +// RequiredNVLinkDomainIndexAnnotation. +func anyPodHasRequiredNVLinkDomainIndex(objs ...client.Object) bool { + for _, obj := range objs { + if _, ok := podTemplateAnnotation(obj, RequiredNVLinkDomainIndexAnnotation); ok { + return true + } + } + return false +} + +// ComputeDomainsForWorkload scans objs' pod templates for the required-nvlink-domain-index +// annotation and returns one ComputeDomain per distinct raw index value present, plus a mapping +// from each raw annotation value to a ComputeDomainRef identifying the ComputeDomain that backs +// it. If no pod template carries the annotation, it returns (nil, nil, nil): no ComputeDomain is +// needed if nothing declares an NVLink domain requirement. +func ComputeDomainsForWorkload(objs ...client.Object) ([]*nvresourcev1beta1.ComputeDomain, map[string]ComputeDomainRef, error) { + if !anyPodHasRequiredNVLinkDomainIndex(objs...) { + return nil, nil, nil + } + + idxMap, err := sanitizeIndices(objs) + if err != nil { + return nil, nil, err + } + + cdByIdx := make(map[int]*nvresourcev1beta1.ComputeDomain, len(idxMap)) + refByRaw := make(map[string]ComputeDomainRef, len(idxMap)) + for raw, idx := range idxMap { + cd, ok := cdByIdx[idx] + if !ok { + cd = computeDomainForIndex(idx) + cdByIdx[idx] = cd + } + refByRaw[raw] = ComputeDomainRef{ + ComputeDomainName: cd.Name, + ChannelName: cd.Spec.Channel.ResourceClaimTemplate.Name, + } + } + + cds := make([]*nvresourcev1beta1.ComputeDomain, 0, len(cdByIdx)) + for _, cd := range cdByIdx { + cds = append(cds, cd) + } + sort.Slice(cds, func(i, j int) bool { return cds[i].Name < cds[j].Name }) + + return cds, refByRaw, nil } func SetComputeDomainToGPUPodResourceClaims( @@ -327,6 +424,12 @@ func iterPodSpecs(itrf iterPodTemplateSpecFunc, objs ...client.Object) { case *batchv1.CronJob: itrf(&ot.Spec.JobTemplate.Spec.Template) default: + // TODO: third-party operator types (e.g. DynamoGraphDeployment) are invisible to + // the NVLink domain-index scan below since their pod templates aren't one of the + // well-known kinds above. When Karta's generic Pod metadata accessor is vendored + // (https://github.com/run-ai/karta/blob/main/pkg/resource/accessor.go#L88), use it + // here to generically extract pod-template annotations from arbitrary object kinds + // so those types can also carry the required-nvlink-domain-index annotation. continue } } diff --git a/src/compute-plane-services/nvca/pkg/dra/dra_test.go b/src/compute-plane-services/nvca/pkg/dra/dra_test.go index 31a3136186..be3fb2c9f5 100644 --- a/src/compute-plane-services/nvca/pkg/dra/dra_test.go +++ b/src/compute-plane-services/nvca/pkg/dra/dra_test.go @@ -185,6 +185,10 @@ func TestTransformNVLinkOptimizedDRAObjects(t *testing.T) { nvlinkDomainPartitionKeyFoo := "x2c26b46b68ffc68ff9x" nvlinkDomainPartitionKeyFooIdx0 := "xbb4eca334f61af3b67x" + // nvlinkDomainPartitionKeyFooIdx1 is the partition key for the second normalized NVLink + // domain index (2) when two distinct required-nvlink-domain-index values are present; + // computed directly rather than hardcoded since it only arises in the multi-index test case. + nvlinkDomainPartitionKeyFooIdx1 := newPartitionKey([]byte("foo2")) newDefaultPrefAffinity := func() *corev1.Affinity { return &corev1.Affinity{ PodAffinity: &corev1.PodAffinity{ @@ -220,19 +224,6 @@ func TestTransformNVLinkOptimizedDRAObjects(t *testing.T) { }, } } - defaultComputeDomain := &nvresourcev1beta1.ComputeDomain{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultComputeDomainName, - }, - Spec: nvresourcev1beta1.ComputeDomainSpec{ - Channel: &nvresourcev1beta1.ComputeDomainChannelSpec{ - ResourceClaimTemplate: nvresourcev1beta1.ComputeDomainResourceClaimTemplate{ - Name: defaultComputeDomainChannelName, - }, - }, - }, - } - computeDomainChannelName := defaultComputeDomainChannelName staticGPUResourceKey := gpuResourceKeys[0] defaultGPULimit := resource.MustParse("2") @@ -252,30 +243,23 @@ func TestTransformNVLinkOptimizedDRAObjects(t *testing.T) { }}, } } + // newDefaultExpPodSpec is the expected spec for a GPU pod with no required-domain-index + // annotation: it only gets preferred NVLink domain affinity, never a ComputeDomain claim, + // since it never declared a cross-node NVLink requirement. newDefaultExpPodSpec := func() corev1.PodSpec { return corev1.PodSpec{ Containers: []corev1.Container{{ Name: "foo", Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{staticGPUResourceKey: defaultGPULimit}, - Claims: []corev1.ResourceClaim{{ - Name: defaultComputeDomainName, - }}, }, }}, InitContainers: []corev1.Container{{ Name: "foo-init", Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{staticGPUResourceKey: defaultGPULimit}, - Claims: []corev1.ResourceClaim{{ - Name: defaultComputeDomainName, - }}, }, }}, - ResourceClaims: []corev1.PodResourceClaim{{ - Name: defaultComputeDomainName, - ResourceClaimTemplateName: &computeDomainChannelName, - }}, Affinity: newDefaultPrefAffinity(), } } @@ -284,6 +268,66 @@ func TestTransformNVLinkOptimizedDRAObjects(t *testing.T) { Labels: map[string]string{NVLinkDomainPartitionLabel: nvlinkDomainPartitionKeyFoo}, } } + // newExpReqPodSpec is the expected spec for a GPU pod with a required-domain-index + // annotation: it gets a claim on the ComputeDomain provisioned for its normalized index, + // plus required (not preferred) NVLink domain affinity keyed off partitionKey. + newExpReqPodSpec := func(cd *nvresourcev1beta1.ComputeDomain, partitionKey string) corev1.PodSpec { + channelName := cd.Spec.Channel.ResourceClaimTemplate.Name + return corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "foo", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{staticGPUResourceKey: defaultGPULimit}, + Claims: []corev1.ResourceClaim{{ + Name: cd.Name, + }}, + }, + }}, + InitContainers: []corev1.Container{{ + Name: "foo-init", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{staticGPUResourceKey: defaultGPULimit}, + Claims: []corev1.ResourceClaim{{ + Name: cd.Name, + }}, + }, + }}, + ResourceClaims: []corev1.PodResourceClaim{{ + Name: cd.Name, + ResourceClaimTemplateName: &channelName, + }}, + Affinity: &corev1.Affinity{ + PodAffinity: &corev1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: NVLinkDomainPartitionLabel, + Operator: metav1.LabelSelectorOpExists, + }, + { + Key: NVLinkDomainPartitionLabel, + Operator: metav1.LabelSelectorOpIn, + Values: []string{partitionKey}, + }, + }, + }, + TopologyKey: GPUCliqueNodeLabel, + }}, + }, + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: GPUCliqueNodeLabel, + Operator: corev1.NodeSelectorOpExists, + }}, + }}, + }, + }, + }, + } + } for _, tt := range []spec{ { @@ -304,9 +348,6 @@ func TestTransformNVLinkOptimizedDRAObjects(t *testing.T) { Spec: newDefaultExpPodSpec(), }, }, - expDRAObjs: []client.Object{ - defaultComputeDomain, - }, }, { name: "cpu pod", @@ -337,9 +378,6 @@ func TestTransformNVLinkOptimizedDRAObjects(t *testing.T) { }, }, }, - expDRAObjs: []client.Object{ - defaultComputeDomain, - }, }, { name: "zero gpu pod", @@ -370,9 +408,6 @@ func TestTransformNVLinkOptimizedDRAObjects(t *testing.T) { }, }, }, - expDRAObjs: []client.Object{ - defaultComputeDomain, - }, }, { name: "all types", @@ -471,12 +506,33 @@ func TestTransformNVLinkOptimizedDRAObjects(t *testing.T) { }, }, }, + }, + { + name: "single pod with req", + objs: []client.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{RequiredNVLinkDomainIndexAnnotation: "0"}, + }, + Spec: newDefaultPodSpec(), + }, + }, + keyToHash: "foo", + expObjs: []client.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{NVLinkDomainPartitionLabel: nvlinkDomainPartitionKeyFooIdx0}, + Annotations: map[string]string{RequiredNVLinkDomainIndexAnnotation: "0"}, + }, + Spec: newExpReqPodSpec(computeDomainForIndex(1), nvlinkDomainPartitionKeyFooIdx0), + }, + }, expDRAObjs: []client.Object{ - defaultComputeDomain, + computeDomainForIndex(1), }, }, { - name: "single pod with req", + name: "two required indices get distinct ComputeDomains", objs: []client.Object{ &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -484,6 +540,12 @@ func TestTransformNVLinkOptimizedDRAObjects(t *testing.T) { }, Spec: newDefaultPodSpec(), }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{RequiredNVLinkDomainIndexAnnotation: "1"}, + }, + Spec: newDefaultPodSpec(), + }, }, keyToHash: "foo", expObjs: []client.Object{ @@ -492,64 +554,19 @@ func TestTransformNVLinkOptimizedDRAObjects(t *testing.T) { Labels: map[string]string{NVLinkDomainPartitionLabel: nvlinkDomainPartitionKeyFooIdx0}, Annotations: map[string]string{RequiredNVLinkDomainIndexAnnotation: "0"}, }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: "foo", - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{staticGPUResourceKey: defaultGPULimit}, - Claims: []corev1.ResourceClaim{{ - Name: defaultComputeDomainName, - }}, - }, - }}, - InitContainers: []corev1.Container{{ - Name: "foo-init", - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{staticGPUResourceKey: defaultGPULimit}, - Claims: []corev1.ResourceClaim{{ - Name: defaultComputeDomainName, - }}, - }, - }}, - ResourceClaims: []corev1.PodResourceClaim{{ - Name: defaultComputeDomainName, - ResourceClaimTemplateName: &computeDomainChannelName, - }}, - Affinity: &corev1.Affinity{ - PodAffinity: &corev1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: NVLinkDomainPartitionLabel, - Operator: metav1.LabelSelectorOpExists, - }, - { - Key: NVLinkDomainPartitionLabel, - Operator: metav1.LabelSelectorOpIn, - Values: []string{nvlinkDomainPartitionKeyFooIdx0}, - }, - }, - }, - TopologyKey: GPUCliqueNodeLabel, - }}, - }, - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{{ - MatchExpressions: []corev1.NodeSelectorRequirement{{ - Key: GPUCliqueNodeLabel, - Operator: corev1.NodeSelectorOpExists, - }}, - }}, - }, - }, - }, + Spec: newExpReqPodSpec(computeDomainForIndex(1), nvlinkDomainPartitionKeyFooIdx0), + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{NVLinkDomainPartitionLabel: nvlinkDomainPartitionKeyFooIdx1}, + Annotations: map[string]string{RequiredNVLinkDomainIndexAnnotation: "1"}, }, + Spec: newExpReqPodSpec(computeDomainForIndex(2), nvlinkDomainPartitionKeyFooIdx1), }, }, expDRAObjs: []client.Object{ - defaultComputeDomain, + computeDomainForIndex(1), + computeDomainForIndex(2), }, }, } { @@ -568,3 +585,68 @@ func TestTransformNVLinkOptimizedDRAObjects(t *testing.T) { }) } } + +func TestComputeDomainsForWorkload(t *testing.T) { + staticGPUResourceKey := gpuResourceKeys[0] + defaultGPULimit := resource.MustParse("2") + newGPUPod := func(annos map[string]string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Annotations: annos}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "foo", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{staticGPUResourceKey: defaultGPULimit}, + }, + }}, + }, + } + } + + t.Run("no annotated objects yields no ComputeDomains", func(t *testing.T) { + cds, refs, err := ComputeDomainsForWorkload(newGPUPod(nil), newGPUPod(map[string]string{"other": "annotation"})) + assert.NoError(t, err) + assert.Nil(t, cds) + assert.Nil(t, refs) + }) + + t.Run("single index yields one ComputeDomain", func(t *testing.T) { + cds, refs, err := ComputeDomainsForWorkload( + newGPUPod(map[string]string{RequiredNVLinkDomainIndexAnnotation: "0"}), + ) + assert.NoError(t, err) + if assert.Len(t, cds, 1) { + assert.Equal(t, computeDomainForIndex(1), cds[0]) + } + assert.Equal(t, map[string]ComputeDomainRef{ + "0": {ComputeDomainName: computeDomainForIndex(1).Name, ChannelName: computeDomainForIndex(1).Spec.Channel.ResourceClaimTemplate.Name}, + }, refs) + }) + + t.Run("distinct indices yield distinct ComputeDomains", func(t *testing.T) { + cds, refs, err := ComputeDomainsForWorkload( + newGPUPod(map[string]string{RequiredNVLinkDomainIndexAnnotation: "0"}), + newGPUPod(map[string]string{RequiredNVLinkDomainIndexAnnotation: "1"}), + newGPUPod(nil), // unannotated pod contributes nothing. + ) + assert.NoError(t, err) + if assert.Len(t, cds, 2) { + assert.Equal(t, computeDomainForIndex(1), cds[0]) + assert.Equal(t, computeDomainForIndex(2), cds[1]) + } + assert.Equal(t, map[string]ComputeDomainRef{ + "0": {ComputeDomainName: computeDomainForIndex(1).Name, ChannelName: computeDomainForIndex(1).Spec.Channel.ResourceClaimTemplate.Name}, + "1": {ComputeDomainName: computeDomainForIndex(2).Name, ChannelName: computeDomainForIndex(2).Spec.Channel.ResourceClaimTemplate.Name}, + }, refs) + }) + + t.Run("repeated raw index reuses the same ComputeDomain", func(t *testing.T) { + cds, refs, err := ComputeDomainsForWorkload( + newGPUPod(map[string]string{RequiredNVLinkDomainIndexAnnotation: "0"}), + newGPUPod(map[string]string{RequiredNVLinkDomainIndexAnnotation: "0"}), + ) + assert.NoError(t, err) + assert.Len(t, cds, 1) + assert.Len(t, refs, 1) + }) +} diff --git a/src/compute-plane-services/nvca/pkg/types/miniservice_types.go b/src/compute-plane-services/nvca/pkg/types/miniservice_types.go index 4121bd5196..a16c6fd1a8 100644 --- a/src/compute-plane-services/nvca/pkg/types/miniservice_types.go +++ b/src/compute-plane-services/nvca/pkg/types/miniservice_types.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" + nvcfdra "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/dra" "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" corev1 "k8s.io/api/core/v1" ) @@ -70,6 +71,13 @@ type MiniserviceMetadata struct { // injected into the ephemeral model-cache-init container by the webhook. // Set only when the ephemeral model-cache backend is selected. ModelCacheInitEnv map[string]string `json:"modelCacheInitEnv,omitempty"` + + // NVLinkComputeDomains maps each raw required-nvlink-domain-index annotation value present + // in the MiniService's workload objects to the ComputeDomain provisioned for it. Populated + // only when NVLinkOptimized is active and at least one workload object carries the + // annotation; the webhook looks up a pod's raw annotation value here rather than computing + // or naming a ComputeDomain itself. + NVLinkComputeDomains map[string]nvcfdra.ComputeDomainRef `json:"nvlinkComputeDomains,omitempty"` } // ToConfigMapData serializes m into ConfigMap-compatible flat string data. diff --git a/src/compute-plane-services/nvca/pkg/webhook/miniservice_mutating_webhook.go b/src/compute-plane-services/nvca/pkg/webhook/miniservice_mutating_webhook.go index e6c7dbd548..1f4cfa1e75 100644 --- a/src/compute-plane-services/nvca/pkg/webhook/miniservice_mutating_webhook.go +++ b/src/compute-plane-services/nvca/pkg/webhook/miniservice_mutating_webhook.go @@ -329,7 +329,7 @@ func (w *miniserviceMutatingWebhook) mutate(ctx context.Context, obj client.Obje if isCreate { // NVLink DRA mutations for claims/scheduling. if w.fff.IsAttributeEnabled(featureflag.AttrNVLinkOptimized) { - w.mutateNVLinkDRA(obj.GetNamespace(), t) + w.mutateNVLinkDRA(obj.GetNamespace(), meta, t) } if _, _, err := w.sharedStorageMutator.mutate(ctx, obj, meta); err != nil { @@ -396,16 +396,23 @@ func (w *miniserviceMutatingWebhook) mutatePodSpec(ps *corev1.PodSpec, meta nvca } } -func (w *miniserviceMutatingWebhook) mutateNVLinkDRA(key string, pod *corev1.Pod) { - // The ComputeDomain's fields are static for the workload so recreating it here is fine. - cd := nvcfdra.NewSingleChannelComputeDomain() - nvcfdra.SetComputeDomainToGPUPodResourceClaims(cd, pod) - annos := pod.GetAnnotations() - if idxStr := annos[nvcfdra.RequiredNVLinkDomainIndexAnnotation]; idxStr != "" { - nvcfdra.SetRequiredNVLinkDomainSchedulingParameters(key, idxStr, pod) - } else { +// mutateNVLinkDRA attaches ComputeDomain-backed NVLink scheduling to pod. A pod only needs a +// ComputeDomain/channel claim if it explicitly declared a required NVLink domain via +// RequiredNVLinkDomainIndexAnnotation: that is the only signal that the pod is doing cross-node +// NVLink memory sharing. The reconciler computes one ComputeDomain per distinct raw annotation +// value present in the MiniService (meta.NVLinkComputeDomains) since a ComputeDomain represents +// a single IMEX domain; the webhook only looks up the ComputeDomain for this pod's value, it +// does not decide naming itself. +func (w *miniserviceMutatingWebhook) mutateNVLinkDRA(key string, meta nvcatypes.MiniserviceMetadata, pod *corev1.Pod) { + idxStr := pod.GetAnnotations()[nvcfdra.RequiredNVLinkDomainIndexAnnotation] + if idxStr == "" { nvcfdra.SetPreferredNVLinkDomainSchedulingParameters(key, pod) + return + } + if ref, ok := meta.NVLinkComputeDomains[idxStr]; ok { + nvcfdra.SetComputeDomainToGPUPodResourceClaims(nvcfdra.ComputeDomainFromRef(ref), pod) } + nvcfdra.SetRequiredNVLinkDomainSchedulingParameters(key, idxStr, pod) } func (w *miniserviceMutatingWebhook) mutateGXCache(lbls map[string]string) { diff --git a/src/compute-plane-services/nvca/pkg/webhook/miniservice_mutating_webhook_test.go b/src/compute-plane-services/nvca/pkg/webhook/miniservice_mutating_webhook_test.go index 28359185a9..950a0eab38 100644 --- a/src/compute-plane-services/nvca/pkg/webhook/miniservice_mutating_webhook_test.go +++ b/src/compute-plane-services/nvca/pkg/webhook/miniservice_mutating_webhook_test.go @@ -1458,3 +1458,65 @@ func volumeMountKeys(vms []corev1.VolumeMount) []string { } return keys } + +func TestMiniserviceMutatingWebhook_MutateNVLinkDRA(t *testing.T) { + newGPUPod := func(annos map[string]string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Annotations: annos}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "app", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1")}, + }, + }}, + }, + } + } + wh := &miniserviceMutatingWebhook{} + + t.Run("no annotation gets preferred affinity, no claim", func(t *testing.T) { + pod := newGPUPod(nil) + wh.mutateNVLinkDRA("ns-key", nvcatypes.MiniserviceMetadata{}, pod) + + assert.Empty(t, pod.Spec.ResourceClaims) + assert.Empty(t, pod.Spec.Containers[0].Resources.Claims) + require.NotNil(t, pod.Spec.Affinity) + require.NotNil(t, pod.Spec.Affinity.PodAffinity) + assert.NotEmpty(t, pod.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution) + assert.Empty(t, pod.Spec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + }) + + t.Run("annotation with matching ref gets claim and required affinity", func(t *testing.T) { + pod := newGPUPod(map[string]string{nvcfdra.RequiredNVLinkDomainIndexAnnotation: "0"}) + meta := nvcatypes.MiniserviceMetadata{ + NVLinkComputeDomains: map[string]nvcfdra.ComputeDomainRef{ + "0": {ComputeDomainName: "nvcf-cd-index-1", ChannelName: "nvcf-cd-channel-1"}, + }, + } + wh.mutateNVLinkDRA("ns-key", meta, pod) + + require.Len(t, pod.Spec.ResourceClaims, 1) + assert.Equal(t, "nvcf-cd-index-1", pod.Spec.ResourceClaims[0].Name) + require.NotNil(t, pod.Spec.ResourceClaims[0].ResourceClaimTemplateName) + assert.Equal(t, "nvcf-cd-channel-1", *pod.Spec.ResourceClaims[0].ResourceClaimTemplateName) + require.Len(t, pod.Spec.Containers[0].Resources.Claims, 1) + assert.Equal(t, "nvcf-cd-index-1", pod.Spec.Containers[0].Resources.Claims[0].Name) + + require.NotNil(t, pod.Spec.Affinity) + require.NotNil(t, pod.Spec.Affinity.PodAffinity) + assert.NotEmpty(t, pod.Spec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + assert.Empty(t, pod.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution) + }) + + t.Run("annotation without matching ref gets required affinity but no claim", func(t *testing.T) { + pod := newGPUPod(map[string]string{nvcfdra.RequiredNVLinkDomainIndexAnnotation: "0"}) + wh.mutateNVLinkDRA("ns-key", nvcatypes.MiniserviceMetadata{}, pod) + + assert.Empty(t, pod.Spec.ResourceClaims) + assert.Empty(t, pod.Spec.Containers[0].Resources.Claims) + require.NotNil(t, pod.Spec.Affinity) + require.NotNil(t, pod.Spec.Affinity.PodAffinity) + assert.NotEmpty(t, pod.Spec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + }) +} From b07f1a426b40a2a144d3c7a3211e366a157ce28a Mon Sep 17 00:00:00 2001 From: vemireddyv Date: Wed, 9 Sep 2026 15:26:02 +0530 Subject: [PATCH 2/2] fix(nvca): declare pkg/dra dependency for the pkg/types Bazel target miniservice_types.go imports pkg/dra, but pkg/types/BUILD.bazel never declared it. Bazel enforces strict dependencies, so the nvca build fails with "missing strict dependencies: ... import of github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/dra". Plain go build does not enforce this, so only the Bazel job catches it. Signed-off-by: vemireddyv --- src/compute-plane-services/nvca/pkg/types/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/src/compute-plane-services/nvca/pkg/types/BUILD.bazel b/src/compute-plane-services/nvca/pkg/types/BUILD.bazel index 91a5915217..4770b83f79 100644 --- a/src/compute-plane-services/nvca/pkg/types/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/types/BUILD.bazel @@ -18,6 +18,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1", + "//src/compute-plane-services/nvca/pkg/dra", "//src/compute-plane-services/nvca/pkg/featureflag", "//src/compute-plane-services/nvca/pkg/nodefeatures/sharedcluster", "//src/compute-plane-services/nvca/pkg/queue",