fix(cloud-tasks): report capacity-shortage error instead of internal ICMS string

src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/CommonRoutineService.java

@@ -50,6 +50,10 @@ class CommonRoutineService {
             "ICMS Request Id %s: Request State %s; Instance Id %s; Instance State %s; ";
     private static final String MESG_HEALTH_INFO_UNAVAILABLE =
             "Health info is not available";
+    private static final String MESG_NO_INSTANCE_PROVISIONED =
+            "No instance could be provisioned for this task before timing out "
+                    + "(commonly caused by cluster capacity exhaustion); "
+                    + "gpu=%s, instanceType=%s, backend=%s";
     private static final String UNKNOWN = "UNKNOWN";
 
     public static final Set<State> TERMINAL_INSTANCE_STATES =
@@ -125,6 +129,15 @@ public static HealthDto getHealthDto(
                                                  errorMessage);
     }
 
+    // Caller-facing message used when ICMS reports no corresponding instance for a Task after
+    // the grace period has elapsed. This is most commonly caused by cluster capacity exhaustion,
+    // but is not asserted as certain since a request that never reached ICMS would look the same.
+    public static String getNoInstanceProvisionedErrorMessage(GpuSpecUdt gpuSpec) {
+        return MESG_NO_INSTANCE_PROVISIONED.formatted(gpuSpec.getGpu(),
+                                                       gpuSpec.getInstanceType(),
+                                                       gpuSpec.getBackend());
+    }
+
     public static String getErrorMessage(Set<HealthDto> healthDtos) {
         var builder = new StringBuilder();
         if (healthDtos.isEmpty()) {

src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/MonitorLaunchedTasksRoutine.java

@@ -20,6 +20,7 @@
 import static com.nvidia.nvct.persistence.task.entity.TaskStatus.LAUNCHED;
 import static com.nvidia.nvct.service.event.EventService.STATUS_CHANGE_EVENT_MESSAGE_WITH_ERROR;
 import static com.nvidia.nvct.service.scheduler.CommonRoutineService.getHealthDto;
+import static com.nvidia.nvct.service.scheduler.CommonRoutineService.getNoInstanceProvisionedErrorMessage;
 import static com.nvidia.nvct.util.NvctConstants.BATCH_SIZE;
 import static com.nvidia.nvct.util.NvctConstants.SPAN_TAG_NCA_ID;
 import static com.nvidia.nvct.util.NvctConstants.SPAN_TAG_TASK_ID;
@@ -75,8 +76,8 @@ public class MonitorLaunchedTasksRoutine {
     private static final String MESG_UNEXPECTED_EXCEPTION =
             "Unexpected exception: {}";
 
-    private static final String MISSING_ICMS_INSTANCES =
-            "No corresponding ICMS instances found";
+    private static final String MESG_NO_ICMS_INSTANCES_FOR_TASK =
+            "Task id '{}': No corresponding ICMS instances found after grace period";
     private static final String NO_HEALTH_INFORMATION_AVAILABLE =
             "No health information available";
 
@@ -212,20 +213,23 @@ private static String getInstanceErrorMessage(List<Instance> instances) {
         return isNotBlank(errorMessage) ? errorMessage : NO_HEALTH_INFORMATION_AVAILABLE;
     }
 
-    // Transitions the Task to ERRORED status when there are no corresponding ICMS Request Id(s).
-    // This should not happen. We saw this when ICMS was not hooked up to NVCT. So, keeping this
-    // for completeness.
+    // Transitions the Task to ERRORED status when ICMS still has no corresponding instance for
+    // this Task after the grace period. This most commonly indicates the cluster ran out of
+    // capacity before ICMS could place the instance, so the caller-facing message reflects that
+    // rather than the internal ICMS request-id lookup that observed it.
     private void transitionToErroredWhenNoIcmsRequestsFound(TaskEntity task) {
         var gpuSpec = task.getGpuSpec();
         var taskId = task.getTaskId();
         var ncaId = task.getNcaId();
-        var healthDto = getHealthDto(gpuSpec, MISSING_ICMS_INSTANCES);
-        log.info(MESG_TASK_HEALTH, taskId, MISSING_ICMS_INSTANCES);
+        var errorMessage = getNoInstanceProvisionedErrorMessage(gpuSpec);
+        var healthDto = getHealthDto(gpuSpec, errorMessage);
+        log.info(MESG_NO_ICMS_INSTANCES_FOR_TASK, taskId);
+        log.info(MESG_TASK_HEALTH, taskId, errorMessage);
 
         taskService.updateTask(taskId, TaskStatus.ERRORED, healthDto);
         taskErrorMetricsService.recordTaskError(ncaId);
         var mesg = STATUS_CHANGE_EVENT_MESSAGE_WITH_ERROR
-                        .formatted(LAUNCHED, ERRORED, MISSING_ICMS_INSTANCES);
+                        .formatted(LAUNCHED, ERRORED, errorMessage);
         log.info(MESG_TASK_INFO, taskId, mesg);
         eventService.insertEvent(ncaId, taskId, mesg);
     }

src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/MonitorQueuedTasksRoutine.java

@@ -21,6 +21,7 @@
 import static com.nvidia.nvct.persistence.task.entity.TaskStatus.QUEUED;
 import static com.nvidia.nvct.service.event.EventService.STATUS_CHANGE_EVENT_MESSAGE_WITH_ERROR;
 import static com.nvidia.nvct.service.scheduler.CommonRoutineService.getHealthDto;
+import static com.nvidia.nvct.service.scheduler.CommonRoutineService.getNoInstanceProvisionedErrorMessage;
 import static com.nvidia.nvct.util.NvctConstants.BATCH_SIZE;
 import static com.nvidia.nvct.util.NvctConstants.SPAN_TAG_NCA_ID;
 import static com.nvidia.nvct.util.NvctConstants.SPAN_TAG_TASK_ID;
@@ -77,8 +78,8 @@ public class MonitorQueuedTasksRoutine {
     private static final String MESG_UNEXPECTED_EXCEPTION =
             "Unexpected exception: {}";
 
-    private static final String MISSING_ICMS_REQUEST_IDS =
-            "No corresponding ICMS request-id(s) found";
+    private static final String MESG_NO_ICMS_INSTANCES_FOR_TASK =
+            "Task id '{}': No corresponding ICMS request-id(s) found after grace period";
     private static final String NO_HEALTH_INFORMATION_AVAILABLE =
             "No health information available";
 
@@ -215,20 +216,23 @@ private void processQueuedTask(TaskEntity taskEntity) {
         log.info(MESG_ICMS_REQUESTS_CAN_PRODUCE, taskId);
     }
 
-    // Transitions the Task to ERRORED status when there are no corresponding ICMS Request Id(s).
-    // This should not happen. We saw this when ICMS was not hooked up to NVCT. So, keeping this
-    // for completeness.
+    // Transitions the Task to ERRORED status when ICMS still has no corresponding instance for
+    // this Task after the grace period. This most commonly indicates the cluster ran out of
+    // capacity before ICMS could place the instance, so the caller-facing message reflects that
+    // rather than the internal ICMS request-id lookup that observed it.
     private void transitionToErroredWhenNoIcmsRequestsFound(TaskEntity taskEntity) {
         var taskId = taskEntity.getTaskId();
         var ncaId = taskEntity.getNcaId();
         var gpuSpec = taskEntity.getGpuSpec();
-        var healthDto = getHealthDto(gpuSpec, MISSING_ICMS_REQUEST_IDS);
-        log.info(MESG_TASK_HEALTH, taskId, MISSING_ICMS_REQUEST_IDS);
+        var errorMessage = getNoInstanceProvisionedErrorMessage(gpuSpec);
+        var healthDto = getHealthDto(gpuSpec, errorMessage);
+        log.info(MESG_NO_ICMS_INSTANCES_FOR_TASK, taskId);
+        log.info(MESG_TASK_HEALTH, taskId, errorMessage);
 
         taskService.updateTask(taskId, TaskStatus.ERRORED, healthDto);
         taskErrorMetricsService.recordTaskError(ncaId);
         var mesg = STATUS_CHANGE_EVENT_MESSAGE_WITH_ERROR
-                        .formatted(QUEUED, ERRORED, MISSING_ICMS_REQUEST_IDS);
+                        .formatted(QUEUED, ERRORED, errorMessage);
         log.info(MESG_TASK_INFO, taskId, mesg);
         eventService.insertEvent(ncaId, taskId, mesg);
     }

src/control-plane-services/cloud-tasks/nvct-core/src/test/java/com/nvidia/nvct/service/scheduler/MonitorLaunchedTasksRoutineTest.java

@@ -195,6 +195,33 @@ void testNoCapacityButNotTimedOutRun() {
         assertThat(events).isEqualTo(updatedEvents);
     }
 
+    @Test
+    @SneakyThrows
+    void testNoIcmsInstancesFoundRun() {
+        // Empty contexts list makes ICMS's GetInstancesByTaskId return an empty Instances list,
+        // distinct from the NO_CAPACITY terminal-instance-state tests above.
+        MockIcmsServer.start(icmsBaseUrl, jsonMapper, List.of());
+
+        testTaskService.createTask(TEST_NCA_ID, TEST_TASK_ID_1, TEST_ICMS_REQ_ID_1);
+        // make sure current task status is LAUNCHED
+        taskService.updateTask(TEST_TASK_ID_1, LAUNCHED);
+
+        // Fake time for testing purposes so lastUpdatedAt is beyond MAX_DURATION.
+        monitorLaunchedTasksRoutine.runUnchecked(Instant.now().plus(Duration.ofMinutes(100)));
+
+        var updatedTask = taskService.fetchTask(TEST_TASK_ID_1);
+        assertThat(updatedTask.getStatus()).isEqualTo(ERRORED);
+        var healthInfo = taskMapperService.deserializeHealth(updatedTask.getHealth())
+                .orElseThrow();
+        assertThat(healthInfo.error()).contains("capacity exhaustion");
+        assertThat(healthInfo.error()).doesNotContain("ICMS request-id");
+        assertThat(healthInfo.error()).doesNotContain(TEST_ICMS_REQ_ID_1.toString());
+
+        var events = eventService.fetchEvents(TEST_NCA_ID, TEST_TASK_ID_1);
+        assertThat(events).isNotNull();
+        assertThat(events.getLast().message()).contains("capacity exhaustion");
+    }
+
     @Test
     @SneakyThrows
     void testHasCapacityRun() {

src/control-plane-services/cloud-tasks/nvct-core/src/test/java/com/nvidia/nvct/service/scheduler/MonitorQueuedTasksRoutineTest.java

@@ -200,6 +200,36 @@ void testNoCapacityButNotTimedOutRun() {
         assertThat(updatedEvents).hasSize(1);
     }
 
+    @Test
+    @SneakyThrows
+    void testNoIcmsInstancesFoundRun() {
+        // Empty contexts list makes ICMS's GetInstancesByTaskId return an empty Instances list,
+        // distinct from the NO_CAPACITY terminal-instance-state tests above.
+        MockIcmsServer.start(icmsBaseUrl, jsonMapper, List.of());
+
+        testTaskService.createTask(TEST_NCA_ID,
+                                   TEST_TASK_ID_1,
+                                   TEST_ICMS_REQ_ID_1,
+                                   Instant.now().minus(Duration.ofMinutes(5)));
+        var taskEntity = tasksRepository.findById(TEST_TASK_ID_1);
+        assertThat(taskEntity).isPresent();
+        assertThat(taskEntity.get().getStatus()).isEqualTo(QUEUED);
+
+        monitorQueuedTasksRoutine.runUnchecked();
+
+        var updatedTask = taskService.fetchTask(TEST_TASK_ID_1);
+        assertThat(updatedTask.getStatus()).isEqualTo(ERRORED);
+        var healthInfo = taskMapperService.deserializeHealth(updatedTask.getHealth())
+                .orElseThrow();
+        assertThat(healthInfo.error()).contains("capacity exhaustion");
+        assertThat(healthInfo.error()).doesNotContain("ICMS request-id");
+        assertThat(healthInfo.error()).doesNotContain(TEST_ICMS_REQ_ID_1.toString());
+
+        var events = eventService.fetchEvents(TEST_NCA_ID, TEST_TASK_ID_1);
+        assertThat(events).isNotNull();
+        assertThat(events.getLast().message()).contains("capacity exhaustion");
+    }
+
     @Test
     @SneakyThrows
     void testHasCapacityRun() {