fix(nvca): gate NVLink ComputeDomain allocation on domain-index annotation

docs/user/cluster-management/topology-aware-scheduling.md

@@ -22,11 +22,13 @@ See the
 and [Dynamo topology guide](https://docs.nvidia.com/dynamo/v1.4.1/kubernetes-deployment/scale/topology-aware-scheduling)
 for component-level details.
 
-Helm functions can also use the legacy
-[`dra.nvcf.nvidia.io` partition annotation](../helm-functions.md#legacy-nvca-nvlink-partition-annotation).
+Helm functions can also use the
+[`dra.nvcf.nvidia.io` partition annotation](../helm-functions.md#required-nvlink-domain-index-annotation).
 That path uses Kubernetes Pod affinity and is best-effort without KAI Scheduler
 or Grove topology-aware scheduling. Use the KAI or Grove mechanisms on this
-page when clique placement must be coordinated for the complete workload.
+page when clique placement must be coordinated for the complete workload. The
+annotation is still required for NVCA to allocate a `ComputeDomain` for a
+function's Pods, even when using KAI or Grove for placement.
 
 ## Prerequisites
 

docs/user/helm-functions.md

@@ -98,10 +98,10 @@ to connect GPU workload Pods through IMEX.
 Each GPU-enabled Pod must request a full node of GPUs.
 </Note>
 
-#### Legacy NVCA NVLink partition annotation
+#### Required NVLink domain index annotation
 
 <Warning>
-NVLink partition placement through these legacy affinity rules is best-effort
+NVLink partition placement through these affinity rules is best-effort
 without KAI Scheduler or Grove topology-aware scheduling. The rules do not
 provide atomic gang placement. Concurrent Pods can initially land in different
 cliques, and distinct logical groups can land in the same clique. Use
@@ -111,8 +111,16 @@ when the workload requires all Pods to fit and start in a specific topology
 domain.
 </Warning>
 
+<Note>
+This annotation is not optional legacy configuration. It is the signal NVCA
+uses to decide whether a Pod needs a `ComputeDomain` and IMEX channel claim
+at all. Set it even for charts that use KAI Scheduler or Grove topology
+constraints for scheduling, if the function needs cross-node NVLink memory
+sharing.
+</Note>
+
 For charts that do not use KAI Scheduler or Grove topology constraints, NVCA
-supports this legacy Pod template annotation:
+supports this Pod template annotation:
 
 ```yaml
 spec:
@@ -136,6 +144,27 @@ rack or clique:
   require the groups to use different GPU cliques.
 - Pods without the annotation share a default logical group.
 
+##### ComputeDomain allocation
+
+NVCA uses this annotation's presence, not just its value, as the signal for
+whether a Pod needs a `ComputeDomain` and IMEX channel resource claim at all:
+
+- A Pod with the annotation set has declared that it needs to be scheduled
+  into a specific NVLink domain, which only makes sense if it does cross-node
+  NVLink memory sharing. NVCA allocates a `ComputeDomain` for that domain and
+  attaches a channel resource claim to the Pod's GPU containers.
+- A Pod without the annotation has no declared NVLink domain requirement. Its
+  GPU peers can already land in different, unrelated NVLink domains, or on
+  nodes with no NVLink domain membership at all. NVCA does not attach a
+  `ComputeDomain` claim to it, since the claim would provide no functional
+  benefit to a Pod that never declared a cross-node NVLink requirement.
+
+NVCA creates one `ComputeDomain` per distinct annotation value present in the
+function, not one shared `ComputeDomain` for the whole function. A
+`ComputeDomain` represents a single IMEX domain, so Pods that set different
+values are meant to join different, independent NVLink domains and each gets
+its own `ComputeDomain` and channel.
+
 On an NVLink-optimized cluster, NVCA mutates each admitted Pod as follows:
 
 - It adds the generated `dra.nvcf.nvidia.io/nvlink-domain-partition` label.
@@ -145,9 +174,10 @@ On an NVLink-optimized cluster, NVCA mutates each admitted Pod as follows:
 - For an unannotated Pod, it adds preferred Pod affinity with weight 100. The
   scheduler can spread these Pods when it cannot satisfy the preference.
 - It requires placement on a node that has the `nvidia.com/gpu.clique` label.
-- It adds the function's `ComputeDomain` resource claim to containers that
-  request `nvidia.com/gpu`, `nvidia.com/pgpu`, `nvidia.com/gpu.shared`, or an
-  `nvidia.com/mig-*` resource.
+- For an annotated Pod, it adds the resource claim for the `ComputeDomain`
+  provisioned for that Pod's annotation value to containers that request
+  `nvidia.com/gpu`, `nvidia.com/pgpu`, `nvidia.com/gpu.shared`, or an
+  `nvidia.com/mig-*` resource. An unannotated Pod never receives this claim.
 
 ## Limitations
 

examples/function-samples/helmchart-samples/multi-node-helm-function-test/multi-node-test/templates/statefulset.yaml

@@ -45,12 +45,13 @@ spec:
     metadata:
       {{- with $profile.podAnnotations }}
       annotations:
-        # Legacy NVCA placement on an NVLink-optimized cluster:
+        # NVCA placement and ComputeDomain allocation on an NVLink-optimized cluster:
         # - Set dra.nvcf.nvidia.io/required-nvlink-domain-index to a logical group index,
-        #   such as "0", for required pod affinity among Pods with the same value.
+        #   such as "0", for required pod affinity among Pods with the same value, and to
+        #   have NVCA allocate a ComputeDomain and IMEX channel claim for this Pod.
         # - Omit dra.nvcf.nvidia.io/required-nvlink-domain-index for preferred
-        #   same-clique placement.
-        # These options are best-effort without KAI Scheduler or Grove
+        #   same-clique placement with no ComputeDomain or channel claim.
+        # Placement through these options is best-effort without KAI Scheduler or Grove
         # topology-aware scheduling. Different indices do not guarantee
         # placement in different physical NVLink partitions.
         {{- toYaml . | nindent 8 }}

src/compute-plane-services/nvca/internal/miniservice/metadata_configmap.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v1alpha1"
 	nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1"
+	nvcfdra "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/dra"
 	"github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/featureflag"
 	"github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nodefeatures"
 	"github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/enforce/kaischeduler"
@@ -50,6 +51,7 @@ type MetadataInput struct {
 	OTelCollectorEnvVars          []corev1.EnvVar
 	TerminationGracePeriodSeconds *int64
 	ModelCacheInitEnv             map[string]string
+	NVLinkComputeDomains          map[string]nvcfdra.ComputeDomainRef
 }
 
 // buildMiniserviceMetadata constructs a MiniserviceMetadata from the controller's
@@ -86,6 +88,7 @@ func (r *Reconciler) buildMiniserviceMetadata(
 		ImagePullSecretNames:          secretNames,
 		TerminationGracePeriodSeconds: in.TerminationGracePeriodSeconds,
 		ModelCacheInitEnv:             in.ModelCacheInitEnv,
+		NVLinkComputeDomains:          in.NVLinkComputeDomains,
 	}
 
 	meta.Labels, meta.Annotations = newGeneralObjectLabelsAndAnnotations(

src/compute-plane-services/nvca/internal/miniservice/reconcile.go

@@ -846,7 +846,14 @@ func (r *Reconciler) doInstall(ctx context.Context,
 	infraObjs = append(infraObjs, utilsPod)
 
 	if r.FeatureFlagFetcher.IsAttributeEnabled(featureflag.AttrNVLinkOptimized) {
-		infraObjs = append(infraObjs, nvcfdra.NewSingleChannelComputeDomain())
+		cds, refs, err := nvcfdra.ComputeDomainsForWorkload(workloadObjs...)
+		if err != nil {
+			return reconcile.Result{}, reconcile.TerminalError(fmt.Errorf("compute NVLink ComputeDomains: %w", err))
+		}
+		for _, cd := range cds {
+			infraObjs = append(infraObjs, cd)
+		}
+		metaInput.NVLinkComputeDomains = refs
 	}
 
 	// Create the miniservice metadata ConfigMap before any objects are created

src/compute-plane-services/nvca/internal/miniservice/reconcile_test.go

@@ -1682,7 +1682,7 @@ func TestReconcile_Function_NVLinkOptimized(t *testing.T) {
 				},
 			},
 		}
-		testReconcileNVLinkOptimizedHelper(t, helmObjs, expDeployments)
+		testReconcileNVLinkOptimizedHelper(t, helmObjs, expDeployments, nil)
 	})
 	t.Run("required domains", func(t *testing.T) {
 		helmObjs := []client.Object{
@@ -1693,12 +1693,14 @@ func TestReconcile_Function_NVLinkOptimized(t *testing.T) {
 				},
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "foo1",
-					Annotations: map[string]string{
-						nvcfdra.RequiredNVLinkDomainIndexAnnotation: "0",
-					},
 				},
 				Spec: appsv1.DeploymentSpec{
 					Template: corev1.PodTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{
+							Annotations: map[string]string{
+								nvcfdra.RequiredNVLinkDomainIndexAnnotation: "0",
+							},
+						},
 						Spec: corev1.PodSpec{
 							Containers: []corev1.Container{{
 								Name:    "test",
@@ -1719,12 +1721,14 @@ func TestReconcile_Function_NVLinkOptimized(t *testing.T) {
 				},
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "foo2",
-					Annotations: map[string]string{
-						nvcfdra.RequiredNVLinkDomainIndexAnnotation: "1",
-					},
 				},
 				Spec: appsv1.DeploymentSpec{
 					Template: corev1.PodTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{
+							Annotations: map[string]string{
+								nvcfdra.RequiredNVLinkDomainIndexAnnotation: "1",
+							},
+						},
 						Spec: corev1.PodSpec{
 							Containers: []corev1.Container{{
 								Name:    "test",
@@ -1741,16 +1745,20 @@ func TestReconcile_Function_NVLinkOptimized(t *testing.T) {
 		}
 		// Workload objects are bare Helm renders; DRA resource claims, NVLink labels,
 		// affinity, and service account are injected by the webhook at Pod admission.
+		// The required-nvlink-domain-index annotation lives on the pod template, since that
+		// is the only location Kubernetes copies down onto the Pods the webhook admits.
 		expDeployments := []appsv1.Deployment{
 			{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "foo1",
-					Annotations: map[string]string{
-						nvcfdra.RequiredNVLinkDomainIndexAnnotation: "0",
-					},
 				},
 				Spec: appsv1.DeploymentSpec{
 					Template: corev1.PodTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{
+							Annotations: map[string]string{
+								nvcfdra.RequiredNVLinkDomainIndexAnnotation: "0",
+							},
+						},
 						Spec: corev1.PodSpec{
 							Containers: []corev1.Container{{
 								Name:    "test",
@@ -1768,12 +1776,14 @@ func TestReconcile_Function_NVLinkOptimized(t *testing.T) {
 			{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "foo2",
-					Annotations: map[string]string{
-						nvcfdra.RequiredNVLinkDomainIndexAnnotation: "1",
-					},
 				},
 				Spec: appsv1.DeploymentSpec{
 					Template: corev1.PodTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{
+							Annotations: map[string]string{
+								nvcfdra.RequiredNVLinkDomainIndexAnnotation: "1",
+							},
+						},
 						Spec: corev1.PodSpec{
 							Containers: []corev1.Container{{
 								Name:    "test",
@@ -1789,11 +1799,16 @@ func TestReconcile_Function_NVLinkOptimized(t *testing.T) {
 				},
 			},
 		}
-		testReconcileNVLinkOptimizedHelper(t, helmObjs, expDeployments)
+		testReconcileNVLinkOptimizedHelper(t, helmObjs, expDeployments, []string{"nvcf-cd-index-1", "nvcf-cd-index-2"})
 	})
 }
 
-func testReconcileNVLinkOptimizedHelper(t *testing.T, helmObjs []client.Object, expDeployments []appsv1.Deployment) {
+func testReconcileNVLinkOptimizedHelper(
+	t *testing.T,
+	helmObjs []client.Object,
+	expDeployments []appsv1.Deployment,
+	expComputeDomainNames []string,
+) {
 	ctx := newTestContext()
 	testScheme := mgrScheme
 
@@ -2154,6 +2169,19 @@ rules:
 		assert.Equal(t, expDeployment, gotDeployment)
 	}
 
+	// One ComputeDomain must be created per distinct required-nvlink-domain-index value present
+	// in the workload objects, and none at all when nothing declares one, since a ComputeDomain
+	// represents a single IMEX domain.
+	gotComputeDomains := &nvresourcev1beta1.ComputeDomainList{}
+	err = r.Client.List(ctx, gotComputeDomains, client.InNamespace(ms.Spec.Namespace))
+	require.NoError(t, err)
+	gotComputeDomainNames := make([]string, len(gotComputeDomains.Items))
+	for i, cd := range gotComputeDomains.Items {
+		gotComputeDomainNames[i] = cd.Name
+	}
+	sort.Strings(gotComputeDomainNames)
+	assert.ElementsMatch(t, expComputeDomainNames, gotComputeDomainNames)
+
 	err = r.Client.Get(ctx, client.ObjectKeyFromObject(ms), ms)
 	require.NoError(t, err)
 	assert.Equal(t, v1alpha1.MiniServiceInstalled, ms.Status.Phase)