diff --git a/src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/CommonRoutineService.java b/src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/CommonRoutineService.java index 03895cc560..ef1c20ec6c 100644 --- a/src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/CommonRoutineService.java +++ b/src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/CommonRoutineService.java @@ -50,6 +50,10 @@ class CommonRoutineService { "ICMS Request Id %s: Request State %s; Instance Id %s; Instance State %s; "; private static final String MESG_HEALTH_INFO_UNAVAILABLE = "Health info is not available"; + private static final String MESG_NO_INSTANCE_PROVISIONED = + "No instance could be provisioned for this task before timing out " + + "(commonly caused by cluster capacity exhaustion); " + + "gpu=%s, instanceType=%s, backend=%s"; private static final String UNKNOWN = "UNKNOWN"; public static final Set TERMINAL_INSTANCE_STATES = @@ -125,6 +129,15 @@ public static HealthDto getHealthDto( errorMessage); } + // Caller-facing message used when ICMS reports no corresponding instance for a Task after + // the grace period has elapsed. This is most commonly caused by cluster capacity exhaustion, + // but is not asserted as certain since a request that never reached ICMS would look the same. + public static String getNoInstanceProvisionedErrorMessage(GpuSpecUdt gpuSpec) { + return MESG_NO_INSTANCE_PROVISIONED.formatted(gpuSpec.getGpu(), + gpuSpec.getInstanceType(), + gpuSpec.getBackend()); + } + public static String getErrorMessage(Set healthDtos) { var builder = new StringBuilder(); if (healthDtos.isEmpty()) { diff --git a/src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/MonitorLaunchedTasksRoutine.java b/src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/MonitorLaunchedTasksRoutine.java index c077ce637e..7132d62ed9 100644 --- a/src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/MonitorLaunchedTasksRoutine.java +++ b/src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/MonitorLaunchedTasksRoutine.java @@ -20,6 +20,7 @@ import static com.nvidia.nvct.persistence.task.entity.TaskStatus.LAUNCHED; import static com.nvidia.nvct.service.event.EventService.STATUS_CHANGE_EVENT_MESSAGE_WITH_ERROR; import static com.nvidia.nvct.service.scheduler.CommonRoutineService.getHealthDto; +import static com.nvidia.nvct.service.scheduler.CommonRoutineService.getNoInstanceProvisionedErrorMessage; import static com.nvidia.nvct.util.NvctConstants.BATCH_SIZE; import static com.nvidia.nvct.util.NvctConstants.SPAN_TAG_NCA_ID; import static com.nvidia.nvct.util.NvctConstants.SPAN_TAG_TASK_ID; @@ -75,8 +76,8 @@ public class MonitorLaunchedTasksRoutine { private static final String MESG_UNEXPECTED_EXCEPTION = "Unexpected exception: {}"; - private static final String MISSING_ICMS_INSTANCES = - "No corresponding ICMS instances found"; + private static final String MESG_NO_ICMS_INSTANCES_FOR_TASK = + "Task id '{}': No corresponding ICMS instances found after grace period"; private static final String NO_HEALTH_INFORMATION_AVAILABLE = "No health information available"; @@ -212,20 +213,23 @@ private static String getInstanceErrorMessage(List instances) { return isNotBlank(errorMessage) ? errorMessage : NO_HEALTH_INFORMATION_AVAILABLE; } - // Transitions the Task to ERRORED status when there are no corresponding ICMS Request Id(s). - // This should not happen. We saw this when ICMS was not hooked up to NVCT. So, keeping this - // for completeness. + // Transitions the Task to ERRORED status when ICMS still has no corresponding instance for + // this Task after the grace period. This most commonly indicates the cluster ran out of + // capacity before ICMS could place the instance, so the caller-facing message reflects that + // rather than the internal ICMS request-id lookup that observed it. private void transitionToErroredWhenNoIcmsRequestsFound(TaskEntity task) { var gpuSpec = task.getGpuSpec(); var taskId = task.getTaskId(); var ncaId = task.getNcaId(); - var healthDto = getHealthDto(gpuSpec, MISSING_ICMS_INSTANCES); - log.info(MESG_TASK_HEALTH, taskId, MISSING_ICMS_INSTANCES); + var errorMessage = getNoInstanceProvisionedErrorMessage(gpuSpec); + var healthDto = getHealthDto(gpuSpec, errorMessage); + log.info(MESG_NO_ICMS_INSTANCES_FOR_TASK, taskId); + log.info(MESG_TASK_HEALTH, taskId, errorMessage); taskService.updateTask(taskId, TaskStatus.ERRORED, healthDto); taskErrorMetricsService.recordTaskError(ncaId); var mesg = STATUS_CHANGE_EVENT_MESSAGE_WITH_ERROR - .formatted(LAUNCHED, ERRORED, MISSING_ICMS_INSTANCES); + .formatted(LAUNCHED, ERRORED, errorMessage); log.info(MESG_TASK_INFO, taskId, mesg); eventService.insertEvent(ncaId, taskId, mesg); } diff --git a/src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/MonitorQueuedTasksRoutine.java b/src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/MonitorQueuedTasksRoutine.java index 51dd4b7ca0..64fc78f3d7 100644 --- a/src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/MonitorQueuedTasksRoutine.java +++ b/src/control-plane-services/cloud-tasks/nvct-core/src/main/java/com/nvidia/nvct/service/scheduler/MonitorQueuedTasksRoutine.java @@ -21,6 +21,7 @@ import static com.nvidia.nvct.persistence.task.entity.TaskStatus.QUEUED; import static com.nvidia.nvct.service.event.EventService.STATUS_CHANGE_EVENT_MESSAGE_WITH_ERROR; import static com.nvidia.nvct.service.scheduler.CommonRoutineService.getHealthDto; +import static com.nvidia.nvct.service.scheduler.CommonRoutineService.getNoInstanceProvisionedErrorMessage; import static com.nvidia.nvct.util.NvctConstants.BATCH_SIZE; import static com.nvidia.nvct.util.NvctConstants.SPAN_TAG_NCA_ID; import static com.nvidia.nvct.util.NvctConstants.SPAN_TAG_TASK_ID; @@ -77,8 +78,8 @@ public class MonitorQueuedTasksRoutine { private static final String MESG_UNEXPECTED_EXCEPTION = "Unexpected exception: {}"; - private static final String MISSING_ICMS_REQUEST_IDS = - "No corresponding ICMS request-id(s) found"; + private static final String MESG_NO_ICMS_INSTANCES_FOR_TASK = + "Task id '{}': No corresponding ICMS request-id(s) found after grace period"; private static final String NO_HEALTH_INFORMATION_AVAILABLE = "No health information available"; @@ -215,20 +216,23 @@ private void processQueuedTask(TaskEntity taskEntity) { log.info(MESG_ICMS_REQUESTS_CAN_PRODUCE, taskId); } - // Transitions the Task to ERRORED status when there are no corresponding ICMS Request Id(s). - // This should not happen. We saw this when ICMS was not hooked up to NVCT. So, keeping this - // for completeness. + // Transitions the Task to ERRORED status when ICMS still has no corresponding instance for + // this Task after the grace period. This most commonly indicates the cluster ran out of + // capacity before ICMS could place the instance, so the caller-facing message reflects that + // rather than the internal ICMS request-id lookup that observed it. private void transitionToErroredWhenNoIcmsRequestsFound(TaskEntity taskEntity) { var taskId = taskEntity.getTaskId(); var ncaId = taskEntity.getNcaId(); var gpuSpec = taskEntity.getGpuSpec(); - var healthDto = getHealthDto(gpuSpec, MISSING_ICMS_REQUEST_IDS); - log.info(MESG_TASK_HEALTH, taskId, MISSING_ICMS_REQUEST_IDS); + var errorMessage = getNoInstanceProvisionedErrorMessage(gpuSpec); + var healthDto = getHealthDto(gpuSpec, errorMessage); + log.info(MESG_NO_ICMS_INSTANCES_FOR_TASK, taskId); + log.info(MESG_TASK_HEALTH, taskId, errorMessage); taskService.updateTask(taskId, TaskStatus.ERRORED, healthDto); taskErrorMetricsService.recordTaskError(ncaId); var mesg = STATUS_CHANGE_EVENT_MESSAGE_WITH_ERROR - .formatted(QUEUED, ERRORED, MISSING_ICMS_REQUEST_IDS); + .formatted(QUEUED, ERRORED, errorMessage); log.info(MESG_TASK_INFO, taskId, mesg); eventService.insertEvent(ncaId, taskId, mesg); } diff --git a/src/control-plane-services/cloud-tasks/nvct-core/src/test/java/com/nvidia/nvct/service/scheduler/MonitorLaunchedTasksRoutineTest.java b/src/control-plane-services/cloud-tasks/nvct-core/src/test/java/com/nvidia/nvct/service/scheduler/MonitorLaunchedTasksRoutineTest.java index a713e2d365..b38e69ff62 100644 --- a/src/control-plane-services/cloud-tasks/nvct-core/src/test/java/com/nvidia/nvct/service/scheduler/MonitorLaunchedTasksRoutineTest.java +++ b/src/control-plane-services/cloud-tasks/nvct-core/src/test/java/com/nvidia/nvct/service/scheduler/MonitorLaunchedTasksRoutineTest.java @@ -195,6 +195,33 @@ void testNoCapacityButNotTimedOutRun() { assertThat(events).isEqualTo(updatedEvents); } + @Test + @SneakyThrows + void testNoIcmsInstancesFoundRun() { + // Empty contexts list makes ICMS's GetInstancesByTaskId return an empty Instances list, + // distinct from the NO_CAPACITY terminal-instance-state tests above. + MockIcmsServer.start(icmsBaseUrl, jsonMapper, List.of()); + + testTaskService.createTask(TEST_NCA_ID, TEST_TASK_ID_1, TEST_ICMS_REQ_ID_1); + // make sure current task status is LAUNCHED + taskService.updateTask(TEST_TASK_ID_1, LAUNCHED); + + // Fake time for testing purposes so lastUpdatedAt is beyond MAX_DURATION. + monitorLaunchedTasksRoutine.runUnchecked(Instant.now().plus(Duration.ofMinutes(100))); + + var updatedTask = taskService.fetchTask(TEST_TASK_ID_1); + assertThat(updatedTask.getStatus()).isEqualTo(ERRORED); + var healthInfo = taskMapperService.deserializeHealth(updatedTask.getHealth()) + .orElseThrow(); + assertThat(healthInfo.error()).contains("capacity exhaustion"); + assertThat(healthInfo.error()).doesNotContain("ICMS request-id"); + assertThat(healthInfo.error()).doesNotContain(TEST_ICMS_REQ_ID_1.toString()); + + var events = eventService.fetchEvents(TEST_NCA_ID, TEST_TASK_ID_1); + assertThat(events).isNotNull(); + assertThat(events.getLast().message()).contains("capacity exhaustion"); + } + @Test @SneakyThrows void testHasCapacityRun() { diff --git a/src/control-plane-services/cloud-tasks/nvct-core/src/test/java/com/nvidia/nvct/service/scheduler/MonitorQueuedTasksRoutineTest.java b/src/control-plane-services/cloud-tasks/nvct-core/src/test/java/com/nvidia/nvct/service/scheduler/MonitorQueuedTasksRoutineTest.java index 50eef30c8c..2fb8519a02 100644 --- a/src/control-plane-services/cloud-tasks/nvct-core/src/test/java/com/nvidia/nvct/service/scheduler/MonitorQueuedTasksRoutineTest.java +++ b/src/control-plane-services/cloud-tasks/nvct-core/src/test/java/com/nvidia/nvct/service/scheduler/MonitorQueuedTasksRoutineTest.java @@ -200,6 +200,36 @@ void testNoCapacityButNotTimedOutRun() { assertThat(updatedEvents).hasSize(1); } + @Test + @SneakyThrows + void testNoIcmsInstancesFoundRun() { + // Empty contexts list makes ICMS's GetInstancesByTaskId return an empty Instances list, + // distinct from the NO_CAPACITY terminal-instance-state tests above. + MockIcmsServer.start(icmsBaseUrl, jsonMapper, List.of()); + + testTaskService.createTask(TEST_NCA_ID, + TEST_TASK_ID_1, + TEST_ICMS_REQ_ID_1, + Instant.now().minus(Duration.ofMinutes(5))); + var taskEntity = tasksRepository.findById(TEST_TASK_ID_1); + assertThat(taskEntity).isPresent(); + assertThat(taskEntity.get().getStatus()).isEqualTo(QUEUED); + + monitorQueuedTasksRoutine.runUnchecked(); + + var updatedTask = taskService.fetchTask(TEST_TASK_ID_1); + assertThat(updatedTask.getStatus()).isEqualTo(ERRORED); + var healthInfo = taskMapperService.deserializeHealth(updatedTask.getHealth()) + .orElseThrow(); + assertThat(healthInfo.error()).contains("capacity exhaustion"); + assertThat(healthInfo.error()).doesNotContain("ICMS request-id"); + assertThat(healthInfo.error()).doesNotContain(TEST_ICMS_REQ_ID_1.toString()); + + var events = eventService.fetchEvents(TEST_NCA_ID, TEST_TASK_ID_1); + assertThat(events).isNotNull(); + assertThat(events.getLast().message()).contains("capacity exhaustion"); + } + @Test @SneakyThrows void testHasCapacityRun() {