diff --git a/dotCMS/src/main/java/com/dotcms/rest/api/v1/maintenance/MaintenanceJobHelper.java b/dotCMS/src/main/java/com/dotcms/rest/api/v1/maintenance/MaintenanceJobHelper.java new file mode 100644 index 00000000000..648e59e0016 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/rest/api/v1/maintenance/MaintenanceJobHelper.java @@ -0,0 +1,133 @@ +package com.dotcms.rest.api.v1.maintenance; + +import com.dotcms.concurrent.DotConcurrentFactory; +import com.dotcms.concurrent.lock.ClusterLockManager; +import com.dotcms.jobs.business.api.JobQueueManagerAPI; +import com.dotcms.jobs.business.job.Job; +import com.dotcms.jobs.business.job.JobPaginatedResult; +import com.dotcms.rest.api.v1.job.JobResponseUtil; +import com.dotcms.rest.api.v1.job.JobStatusResponse; +import com.dotcms.rest.exception.ConflictException; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotRuntimeException; +import com.liferay.portal.model.User; +import java.util.HashMap; +import java.util.Map; +import javax.enterprise.context.ApplicationScoped; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; + +/** + * Creates fix-assets and clean-assets jobs on {@link JobQueueManagerAPI}. A cluster lock + * around the {@code getActiveJobs} check-and-create gives strict 409 semantics: two nodes + * cannot both pass the active-job check in the same millisecond. + * + * @author hassandotcms + */ +@ApplicationScoped +public class MaintenanceJobHelper { + + public static final String FIX_ASSETS_QUEUE = "maintenanceFixAssets"; + public static final String CLEAN_ASSETS_QUEUE = "maintenanceCleanAssets"; + + private static final String JOB_STATUS_PATH = "/api/v1/jobs/%s/status"; + private static final String PARAM_USER_ID = "userId"; + private static final String PARAM_REMOTE_ADDR = "remoteAddr"; + + private final JobQueueManagerAPI jobQueueManagerAPI; + + @Inject + public MaintenanceJobHelper(final JobQueueManagerAPI jobQueueManagerAPI) { + this.jobQueueManagerAPI = jobQueueManagerAPI; + } + + /** + * CDI requires a no-arg constructor for proxy creation. + */ + public MaintenanceJobHelper() { + this.jobQueueManagerAPI = null; + } + + /** + * Creates a fix-assets job, rejecting with 409 Conflict if one is already running. + */ + public JobStatusResponse createFixAssetsJob(final User user, final HttpServletRequest request) { + return createSingletonJob(FIX_ASSETS_QUEUE, user, request, "fix-assets"); + } + + /** + * Creates a clean-orphan-assets job, rejecting with 409 Conflict if one is already running. + */ + public JobStatusResponse createCleanAssetsJob(final User user, final HttpServletRequest request) { + return createSingletonJob(CLEAN_ASSETS_QUEUE, user, request, "clean-assets"); + } + + /** + * Returns the latest job for a given queue — active if one is pending or running, otherwise + * the most recent completed job. Returns {@code null} if no job has ever been created. + */ + public Job getLatestJob(final String queueName) { + try { + // getJobs() orders by created_at DESC — the first row is the most recent job + // across all states (pending, running, completed, failed, canceled). + final JobPaginatedResult result = jobQueueManagerAPI.getJobs(queueName, 1, 1); + return result.jobs().isEmpty() ? null : result.jobs().get(0); + } catch (final DotDataException e) { + throw new DotRuntimeException("Failed to fetch latest job for queue " + queueName, e); + } + } + + /** + * Acquires a short-lived cluster lock keyed on the queue name, re-checks the queue for any + * active job, and enqueues a new job atomically. If the lock cannot be acquired (another node + * is starting a job in this exact moment) or an active job exists, throws + * {@link ConflictException}. + */ + private JobStatusResponse createSingletonJob( + final String queueName, + final User user, + final HttpServletRequest request, + final String humanName) { + + final ClusterLockManager lock = + DotConcurrentFactory.getInstance().getClusterLockManager(queueName); + + final JobStatusResponse result; + try { + result = lock.tryClusterLock(() -> + checkAndEnqueue(queueName, user, request, humanName)); + } catch (final ConflictException e) { + throw e; + } catch (final Throwable t) { + throw new DotRuntimeException("Failed to create " + humanName + " job", t); + } + + if (result == null) { + throw new ConflictException(String.format( + "Another node is currently starting a %s job; retry in a moment", + humanName)); + } + return result; + } + + private JobStatusResponse checkAndEnqueue( + final String queueName, + final User user, + final HttpServletRequest request, + final String humanName) throws DotDataException { + + final JobPaginatedResult active = jobQueueManagerAPI.getActiveJobs(queueName, 1, 1); + if (!active.jobs().isEmpty()) { + throw new ConflictException(String.format( + "A %s job is already running (jobId=%s)", + humanName, active.jobs().get(0).id())); + } + + final Map params = new HashMap<>(); + params.put(PARAM_USER_ID, user.getUserId()); + params.put(PARAM_REMOTE_ADDR, request.getRemoteAddr()); + + final String jobId = jobQueueManagerAPI.createJob(queueName, params); + return JobResponseUtil.buildJobStatusResponse(jobId, JOB_STATUS_PATH, request); + } +} diff --git a/dotCMS/src/main/java/com/dotcms/rest/api/v1/maintenance/MaintenanceResource.java b/dotCMS/src/main/java/com/dotcms/rest/api/v1/maintenance/MaintenanceResource.java index 2677fd73730..1f2f3ed80bf 100644 --- a/dotCMS/src/main/java/com/dotcms/rest/api/v1/maintenance/MaintenanceResource.java +++ b/dotCMS/src/main/java/com/dotcms/rest/api/v1/maintenance/MaintenanceResource.java @@ -12,6 +12,7 @@ import com.dotcms.rest.WebResource; import com.dotcms.rest.annotation.NoCache; import com.dotcms.rest.exception.BadRequestException; +import com.dotcms.rest.exception.ConflictException; import com.dotcms.util.ConversionUtils; import com.dotcms.util.DbExporterUtil; import com.dotcms.util.SizeUtil; @@ -76,6 +77,10 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import com.dotcms.cdi.CDIUtils; +import com.dotcms.jobs.business.job.Job; +import com.dotcms.rest.ResponseEntityJobStatusView; + /** * This REST Endpoint exposes all the different features displayed in the Maintenance portlet * inside the dotCMS back-end. @@ -93,6 +98,13 @@ public class MaintenanceResource implements Serializable { protected static final Lazy ALLOW_DOTCMS_SHUTDOWN_FROM_CONSOLE = Lazy.of(() -> Config.getBooleanProperty("ALLOW_DOTCMS_SHUTDOWN_FROM_CONSOLE", true)); + /** + * Resolved lazily via CDI the first time a fix/clean-assets endpoint is invoked. We avoid + * constructor injection so the no-arg and {@code @VisibleForTesting} constructors used by + * Jersey and existing integration tests keep working unchanged. + */ + private volatile MaintenanceJobHelper jobHelper; + /** * Default class constructor. */ @@ -718,6 +730,179 @@ public final ResponseEntityStringView deletePushedAssets( return new ResponseEntityStringView("success"); } + // ------------------------------------------------------------------------- + // Fix Assets & Clean Assets endpoints — backed by JobQueueManagerAPI + // ------------------------------------------------------------------------- + + /** + * Enqueues a fix-assets job. Runs all registered FixTask classes asynchronously on the + * cluster's job queue. Returns immediately with a job id; poll the job status via + * {@code GET /api/v1/jobs/{jobId}/status} or {@link #getLatestFixAssetsJob}. + */ + @Operation( + summary = "Request a fix-assets job", + description = "Enqueues a fix-assets inconsistencies job on the cluster job queue. " + + "Returns immediately with {jobId, statusUrl}. Rejects with 409 Conflict if " + + "a fix-assets job is already pending or running anywhere in the cluster." + ) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", + description = "Job enqueued", + content = @Content(mediaType = "application/json", + schema = @Schema(implementation = ResponseEntityJobStatusView.class))), + @ApiResponse(responseCode = "401", + description = "Unauthorized - authentication required", + content = @Content(mediaType = "application/json")), + @ApiResponse(responseCode = "403", + description = "Forbidden - CMS Administrator role required", + content = @Content(mediaType = "application/json")), + @ApiResponse(responseCode = "409", + description = "Conflict - a fix-assets job is already running", + content = @Content(mediaType = "application/json")) + }) + @POST + @Path("/assets/_fix") + @NoCache + @Produces({MediaType.APPLICATION_JSON}) + public ResponseEntityJobStatusView requestFixAssetsJob( + @Parameter(hidden = true) @Context final HttpServletRequest request, + @Parameter(hidden = true) @Context final HttpServletResponse response) { + + final User user = assertBackendUser(request, response).getUser(); + return new ResponseEntityJobStatusView( + jobHelper().createFixAssetsJob(user, request)); + } + + /** + * Returns the most recent fix-assets job — the currently active one if any, otherwise the + * most recently completed. Intended for "page reload" or "open in a second tab" scenarios + * where the client has lost the original job id. + */ + @Operation( + summary = "Get latest fix-assets job", + description = "Returns the most recent fix-assets job (active, or most recently " + + "completed). Returns null entity if no fix-assets job has ever run." + ) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", + description = "Latest job status", + content = @Content(mediaType = "application/json", + schema = @Schema(type = "object", + description = "ResponseEntityView wrapping the latest Job " + + "(id, state, progress, result) or null if none exists"))), + @ApiResponse(responseCode = "401", + description = "Unauthorized - authentication required", + content = @Content(mediaType = "application/json")), + @ApiResponse(responseCode = "403", + description = "Forbidden - CMS Administrator role required", + content = @Content(mediaType = "application/json")) + }) + @GET + @Path("/assets/_fix") + @JSONP + @NoCache + @Produces({MediaType.APPLICATION_JSON}) + public ResponseEntityView getLatestFixAssetsJob( + @Parameter(hidden = true) @Context final HttpServletRequest request, + @Parameter(hidden = true) @Context final HttpServletResponse response) { + + assertBackendUser(request, response); + return new ResponseEntityView<>( + jobHelper().getLatestJob(MaintenanceJobHelper.FIX_ASSETS_QUEUE)); + } + + /** + * Enqueues a clean-assets job. The job walks the assets directory and deletes orphan + * binary folders whose contentlet inode is no longer in the database. Returns immediately + * with a job id; poll the job status via {@code GET /api/v1/jobs/{jobId}/status} or + * {@link #getLatestCleanAssetsJob}. + */ + @Operation( + summary = "Request a clean-assets job", + description = "Enqueues a clean orphan assets job on the cluster job queue. Returns " + + "immediately with {jobId, statusUrl}. Rejects with 409 Conflict if a " + + "clean-assets job is already pending or running anywhere in the cluster." + ) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", + description = "Job enqueued", + content = @Content(mediaType = "application/json", + schema = @Schema(implementation = ResponseEntityJobStatusView.class))), + @ApiResponse(responseCode = "401", + description = "Unauthorized - authentication required", + content = @Content(mediaType = "application/json")), + @ApiResponse(responseCode = "403", + description = "Forbidden - CMS Administrator role required", + content = @Content(mediaType = "application/json")), + @ApiResponse(responseCode = "409", + description = "Conflict - a clean-assets job is already running", + content = @Content(mediaType = "application/json")) + }) + @POST + @Path("/assets/_clean") + @NoCache + @Produces({MediaType.APPLICATION_JSON}) + public ResponseEntityJobStatusView requestCleanAssetsJob( + @Parameter(hidden = true) @Context final HttpServletRequest request, + @Parameter(hidden = true) @Context final HttpServletResponse response) { + + final User user = assertBackendUser(request, response).getUser(); + return new ResponseEntityJobStatusView( + jobHelper().createCleanAssetsJob(user, request)); + } + + /** + * Returns the most recent clean-assets job — active if any, otherwise most recently + * completed. Intended for page-reload / second-tab scenarios. + */ + @Operation( + summary = "Get latest clean-assets job", + description = "Returns the most recent clean-assets job (active, or most recently " + + "completed). Returns null entity if no clean-assets job has ever run." + ) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", + description = "Latest job status", + content = @Content(mediaType = "application/json", + schema = @Schema(type = "object", + description = "ResponseEntityView wrapping the latest Job " + + "(id, state, progress, result) or null if none exists"))), + @ApiResponse(responseCode = "401", + description = "Unauthorized - authentication required", + content = @Content(mediaType = "application/json")), + @ApiResponse(responseCode = "403", + description = "Forbidden - CMS Administrator role required", + content = @Content(mediaType = "application/json")) + }) + @GET + @Path("/assets/_clean") + @JSONP + @NoCache + @Produces({MediaType.APPLICATION_JSON}) + public ResponseEntityView getLatestCleanAssetsJob( + @Parameter(hidden = true) @Context final HttpServletRequest request, + @Parameter(hidden = true) @Context final HttpServletResponse response) { + + assertBackendUser(request, response); + return new ResponseEntityView<>( + jobHelper().getLatestJob(MaintenanceJobHelper.CLEAN_ASSETS_QUEUE)); + } + + /** + * Lazily resolves the {@link MaintenanceJobHelper} via CDI on first use. Resource + * instances are constructed by Jersey without CDI injection, so we pull the bean on + * demand. The field is volatile so the double-checked assignment is safe. + */ + private MaintenanceJobHelper jobHelper() { + MaintenanceJobHelper local = jobHelper; + if (local == null) { + local = CDIUtils.getBean(MaintenanceJobHelper.class).orElseThrow(() -> + new DotRuntimeException("MaintenanceJobHelper CDI bean not available")); + jobHelper = local; + } + return local; + } + /** * Verifies that calling user is a backend user required to access the Maintenance portlet. * diff --git a/dotCMS/src/main/java/com/dotcms/rest/api/v1/maintenance/jobs/CleanAssetsJobProcessor.java b/dotCMS/src/main/java/com/dotcms/rest/api/v1/maintenance/jobs/CleanAssetsJobProcessor.java new file mode 100644 index 00000000000..eb1a38586ef --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/rest/api/v1/maintenance/jobs/CleanAssetsJobProcessor.java @@ -0,0 +1,105 @@ +package com.dotcms.rest.api.v1.maintenance.jobs; + +import com.dotcms.jobs.business.error.JobProcessingException; +import com.dotcms.jobs.business.job.Job; +import com.dotcms.jobs.business.processor.JobProcessor; +import com.dotcms.jobs.business.processor.NoRetryPolicy; +import com.dotcms.jobs.business.processor.Queue; +import com.dotcms.rest.api.v1.maintenance.MaintenanceJobHelper; +import com.dotmarketing.portlets.cmsmaintenance.util.CleanAssetsThread; +import com.dotmarketing.portlets.cmsmaintenance.util.CleanAssetsThread.BasicProcessStatus; +import com.dotmarketing.util.Logger; +import com.dotmarketing.util.SecurityLogger; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import javax.enterprise.context.Dependent; + +/** + * Job processor that runs the {@link CleanAssetsThread} walk under the + * {@link com.dotcms.jobs.business.api.JobQueueManagerAPI} framework. + * + *

Legacy {@code CleanAssetsThread} is invoked via {@link Thread#run()} — which is already + * {@code public} on the {@code Thread} class — executing the walk synchronously on the job + * queue's worker thread. No new threads are spawned and no legacy code is modified. DWR + * callers, which still invoke {@link CleanAssetsThread#start()}, are unaffected.

+ * + *

A lightweight scheduled reporter bridges {@link BasicProcessStatus} progress (updated + * inside the walk) into the job's {@code progressTracker} so the standard job-status + * endpoint can surface progress.

+ * + * @author hassandotcms + */ +@Queue(MaintenanceJobHelper.CLEAN_ASSETS_QUEUE) +@NoRetryPolicy +@Dependent +public class CleanAssetsJobProcessor implements JobProcessor { + + private Map resultMetadata = new HashMap<>(); + + @Override + public void process(final Job job) throws JobProcessingException { + + final String userId = String.valueOf(job.parameters().get("userId")); + final String remoteAddr = String.valueOf(job.parameters().get("remoteAddr")); + + SecurityLogger.logInfo(this.getClass(), String.format( + "User '%s' running clean orphan assets (jobId=%s) from ip: %s", + userId, job.id(), remoteAddr)); + Logger.info(this, String.format( + "Executing clean-assets job %s for user %s", job.id(), userId)); + + final CleanAssetsThread worker = CleanAssetsThread.getInstance(true, true); + final BasicProcessStatus status = worker.getProcessStatus(); + + final ScheduledExecutorService reporter = + Executors.newSingleThreadScheduledExecutor(r -> { + final Thread t = new Thread(r, "clean-assets-progress-" + job.id()); + t.setDaemon(true); + return t; + }); + final ScheduledFuture reporterTask = reporter.scheduleAtFixedRate(() -> { + final int total = status.getTotalFiles(); + if (total > 0) { + final float pct = Math.min( + 1.0f, Math.max(0.0f, status.getCurrentFiles() / (float) total)); + job.progressTracker().ifPresent(tracker -> tracker.updateProgress(pct)); + } + }, 1, 1, TimeUnit.SECONDS); + + try { + // Synchronous invocation of run() on THIS thread (the job-queue worker), + // NOT Thread.start(). Executes deleteAssetsWithNoInode() in-line. + worker.run(); + } catch (final Exception e) { + Logger.error(this, "Clean-assets job " + job.id() + " failed", e); + throw new JobProcessingException( + job.id(), "Clean-assets failed: " + e.getMessage(), e); + } finally { + reporterTask.cancel(false); + reporter.shutdown(); + } + + final Map metadata = new HashMap<>(); + metadata.put("totalFiles", status.getTotalFiles()); + metadata.put("currentFiles", status.getCurrentFiles()); + metadata.put("deleted", status.getDeleted()); + metadata.put("finalStatus", status.getStatus()); + this.resultMetadata = metadata; + + job.progressTracker().ifPresent(tracker -> tracker.updateProgress(1.0f)); + + Logger.info(this, String.format( + "Clean-assets job %s completed; deleted=%d totalFiles=%d finalStatus=%s", + job.id(), status.getDeleted(), status.getTotalFiles(), status.getStatus())); + } + + @Override + public Map getResultMetadata(final Job job) { + return resultMetadata.isEmpty() ? Collections.emptyMap() : resultMetadata; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/rest/api/v1/maintenance/jobs/FixAssetsJobProcessor.java b/dotCMS/src/main/java/com/dotcms/rest/api/v1/maintenance/jobs/FixAssetsJobProcessor.java new file mode 100644 index 00000000000..a8ef338c64a --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/rest/api/v1/maintenance/jobs/FixAssetsJobProcessor.java @@ -0,0 +1,78 @@ +package com.dotcms.rest.api.v1.maintenance.jobs; + +import com.dotcms.jobs.business.error.JobProcessingException; +import com.dotcms.jobs.business.job.Job; +import com.dotcms.jobs.business.processor.JobProcessor; +import com.dotcms.jobs.business.processor.NoRetryPolicy; +import com.dotcms.jobs.business.processor.Queue; +import com.dotcms.rest.api.v1.maintenance.MaintenanceJobHelper; +import com.dotmarketing.fixtask.FixTasksExecutor; +import com.dotmarketing.util.Logger; +import com.dotmarketing.util.SecurityLogger; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.enterprise.context.Dependent; + +/** + * Job processor that runs {@link FixTasksExecutor#execute(org.quartz.JobExecutionContext)} + * under the {@link com.dotcms.jobs.business.api.JobQueueManagerAPI} framework. + * + *

Legacy behavior is preserved: the processor calls the existing singleton executor, + * so DWR callers, Quartz-triggered runs, and any other invocation paths continue to work + * unchanged. Only the REST entry point is migrated to the job queue.

+ * + *

Single-in-flight semantics are enforced at job-creation time by + * {@link MaintenanceJobHelper}; the queue itself guarantees single-node execution.

+ * + * @author hassandotcms + */ +@Queue(MaintenanceJobHelper.FIX_ASSETS_QUEUE) +@NoRetryPolicy +@Dependent +@SuppressWarnings("rawtypes") +public class FixAssetsJobProcessor implements JobProcessor { + + private Map resultMetadata = new HashMap<>(); + + @Override + public void process(final Job job) throws JobProcessingException { + + final String userId = String.valueOf(job.parameters().get("userId")); + final String remoteAddr = String.valueOf(job.parameters().get("remoteAddr")); + + SecurityLogger.logInfo(this.getClass(), String.format( + "User '%s' running fix assets inconsistencies (jobId=%s) from ip: %s", + userId, job.id(), remoteAddr)); + Logger.info(this, String.format( + "Executing fix-assets job %s for user %s", job.id(), userId)); + + try { + final FixTasksExecutor executor = FixTasksExecutor.getInstance(); + executor.execute(null); + + final List results = new ArrayList<>(executor.getTasksresults()); + final Map metadata = new HashMap<>(); + metadata.put("tasksRun", results.size()); + metadata.put("results", results); + this.resultMetadata = metadata; + + job.progressTracker().ifPresent(tracker -> tracker.updateProgress(1.0f)); + + Logger.info(this, String.format( + "Fix-assets job %s completed; %d task(s) produced results", + job.id(), results.size())); + } catch (final Exception e) { + Logger.error(this, "Fix-assets job " + job.id() + " failed", e); + throw new JobProcessingException( + job.id(), "Fix-assets failed: " + e.getMessage(), e); + } + } + + @Override + public Map getResultMetadata(final Job job) { + return resultMetadata.isEmpty() ? Collections.emptyMap() : resultMetadata; + } +} diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/cmsmaintenance/util/CleanAssetsThread.java b/dotCMS/src/main/java/com/dotmarketing/portlets/cmsmaintenance/util/CleanAssetsThread.java index bc5961f0ee5..6604ace0c94 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/cmsmaintenance/util/CleanAssetsThread.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/cmsmaintenance/util/CleanAssetsThread.java @@ -19,11 +19,11 @@ */ public class CleanAssetsThread extends Thread { public static class BasicProcessStatus { - private int totalFiles=0; - private int currentFiles=0; - private int deleted=0; - private boolean running=false; - private String status=""; + private volatile int totalFiles=0; + private volatile int currentFiles=0; + private volatile int deleted=0; + private volatile boolean running=false; + private volatile String status=""; public int getDeleted() { return deleted; @@ -73,7 +73,7 @@ public Map buildStatusMap() { * @param restartIfDied creates or not a new instance depending of the current thread is alive or not * @return */ - public static CleanAssetsThread getInstance ( Boolean restartIfDied, boolean processBinary ) { + public static synchronized CleanAssetsThread getInstance ( Boolean restartIfDied, boolean processBinary ) { if ( instance == null ) { diff --git a/dotCMS/src/main/webapp/WEB-INF/openapi/openapi.yaml b/dotCMS/src/main/webapp/WEB-INF/openapi/openapi.yaml index e80406cfb37..b8b3aa24e31 100644 --- a/dotCMS/src/main/webapp/WEB-INF/openapi/openapi.yaml +++ b/dotCMS/src/main/webapp/WEB-INF/openapi/openapi.yaml @@ -11546,6 +11546,110 @@ paths: description: default response tags: - Maintenance + /v1/maintenance/assets/_clean: + get: + description: "Returns the most recent clean-assets job (active, or most recently\ + \ completed). Returns null entity if no clean-assets job has ever run." + operationId: getLatestCleanAssetsJob + responses: + "200": + content: + application/json: + schema: + type: object + description: "ResponseEntityView wrapping the latest Job (id, state,\ + \ progress, result) or null if none exists" + description: Latest job status + "401": + content: + application/json: {} + description: Unauthorized - authentication required + "403": + content: + application/json: {} + description: Forbidden - CMS Administrator role required + summary: Get latest clean-assets job + tags: + - Maintenance + post: + description: "Enqueues a clean orphan assets job on the cluster job queue. Returns\ + \ immediately with {jobId, statusUrl}. Rejects with 409 Conflict if a clean-assets\ + \ job is already pending or running anywhere in the cluster." + operationId: requestCleanAssetsJob + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/ResponseEntityJobStatusView" + description: Job enqueued + "401": + content: + application/json: {} + description: Unauthorized - authentication required + "403": + content: + application/json: {} + description: Forbidden - CMS Administrator role required + "409": + content: + application/json: {} + description: Conflict - a clean-assets job is already running + summary: Request a clean-assets job + tags: + - Maintenance + /v1/maintenance/assets/_fix: + get: + description: "Returns the most recent fix-assets job (active, or most recently\ + \ completed). Returns null entity if no fix-assets job has ever run." + operationId: getLatestFixAssetsJob + responses: + "200": + content: + application/json: + schema: + type: object + description: "ResponseEntityView wrapping the latest Job (id, state,\ + \ progress, result) or null if none exists" + description: Latest job status + "401": + content: + application/json: {} + description: Unauthorized - authentication required + "403": + content: + application/json: {} + description: Forbidden - CMS Administrator role required + summary: Get latest fix-assets job + tags: + - Maintenance + post: + description: "Enqueues a fix-assets inconsistencies job on the cluster job queue.\ + \ Returns immediately with {jobId, statusUrl}. Rejects with 409 Conflict if\ + \ a fix-assets job is already pending or running anywhere in the cluster." + operationId: requestFixAssetsJob + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/ResponseEntityJobStatusView" + description: Job enqueued + "401": + content: + application/json: {} + description: Unauthorized - authentication required + "403": + content: + application/json: {} + description: Forbidden - CMS Administrator role required + "409": + content: + application/json: {} + description: Conflict - a fix-assets job is already running + summary: Request a fix-assets job + tags: + - Maintenance /v1/menu: get: operationId: getMenus diff --git a/dotcms-integration/src/test/java/com/dotcms/rest/api/v1/maintenance/MaintenanceResourceIntegrationTest.java b/dotcms-integration/src/test/java/com/dotcms/rest/api/v1/maintenance/MaintenanceResourceIntegrationTest.java index 4beef0e3c15..814d63c4d85 100644 --- a/dotcms-integration/src/test/java/com/dotcms/rest/api/v1/maintenance/MaintenanceResourceIntegrationTest.java +++ b/dotcms-integration/src/test/java/com/dotcms/rest/api/v1/maintenance/MaintenanceResourceIntegrationTest.java @@ -8,33 +8,53 @@ import com.dotcms.IntegrationTestBase; import com.dotcms.datagen.TestUserUtils; import com.dotcms.datagen.UserDataGen; +import com.dotcms.jobs.business.api.JobQueueManagerAPI; +import com.dotcms.jobs.business.job.Job; +import com.dotcms.jobs.business.job.JobResult; +import com.dotcms.jobs.business.job.JobState; import com.dotcms.mock.request.MockAttributeRequest; import com.dotcms.mock.request.MockHttpRequestIntegrationTest; import com.dotcms.mock.response.MockHttpResponse; +import com.dotcms.rest.ResponseEntityJobStatusView; import com.dotcms.rest.ResponseEntityStringView; +import com.dotcms.rest.ResponseEntityView; +import com.dotcms.rest.api.v1.job.JobStatusResponse; import com.dotcms.rest.exception.BadRequestException; +import com.dotcms.rest.exception.ConflictException; import com.dotcms.rest.exception.SecurityException; import com.dotcms.rest.exception.ValidationException; import com.dotcms.util.IntegrationTestInitService; import com.dotmarketing.business.APILocator; +import com.dotmarketing.util.UUIDGenerator; import com.liferay.portal.model.User; import com.liferay.portal.util.WebKeys; +import java.io.File; +import java.nio.file.Files; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.io.FileUtils; import org.junit.BeforeClass; +import org.junit.FixMethodOrder; import org.junit.Test; +import org.junit.runners.MethodSorters; /** * Integration tests for the maintenance tools REST endpoints in {@link MaintenanceResource}. * * @author hassandotcms */ +@FixMethodOrder(MethodSorters.NAME_ASCENDING) public class MaintenanceResourceIntegrationTest extends IntegrationTestBase { private static MaintenanceResource resource; private static HttpServletResponse mockResponse; private static User adminUser; private static User nonAdminUser; + private static JobQueueManagerAPI jobQueueManagerAPI; @BeforeClass public static void prepare() throws Exception { @@ -47,6 +67,12 @@ public static void prepare() throws Exception { nonAdminUser = new UserDataGen().nextPersisted(); APILocator.getRoleAPI().addRoleToUser( APILocator.getRoleAPI().loadBackEndUserRole(), nonAdminUser); + + jobQueueManagerAPI = APILocator.getJobQueueManagerAPI(); + if (!jobQueueManagerAPI.isStarted()) { + jobQueueManagerAPI.start(); + jobQueueManagerAPI.awaitStart(10, TimeUnit.SECONDS); + } } // ==================== POST /_searchAndReplace ==================== @@ -164,6 +190,227 @@ public void test_deletePushedAssets_asNonAdmin_throwsSecurity() { resource.deletePushedAssets(request, mockResponse); } + // ==================== POST /assets/_fix ==================== + + /** + * Given scenario: an admin user requests a fix-assets job + * Expected result: a job is enqueued, reaches SUCCESS, and its metadata records + * {@code tasksRun} and a {@code results} list + */ + @Test + public void test_requestFixAssetsJob_asAdmin_enqueuesAndCompletes() throws Exception { + final HttpServletRequest request = createAdminRequest(); + + final ResponseEntityJobStatusView view = + resource.requestFixAssetsJob(request, mockResponse); + + assertNotNull(view); + final JobStatusResponse status = view.getEntity(); + assertNotNull(status); + assertNotNull("jobId must be present", status.jobId()); + assertTrue("statusUrl must point to the generic jobs endpoint", + status.statusUrl().contains("/api/v1/jobs/" + status.jobId() + "/status")); + + final Job completed = awaitJobCompletion(status.jobId(), 60); + assertEquals("Fix-assets job should complete successfully", + JobState.SUCCESS, completed.state()); + + final Map metadata = extractMetadata(completed); + assertTrue("fix-assets metadata should record tasksRun", + metadata.containsKey("tasksRun")); + assertTrue("tasksRun must be a non-negative integer", + ((Number) metadata.get("tasksRun")).intValue() >= 0); + assertTrue("fix-assets metadata should include a results list", + metadata.get("results") instanceof List); + } + + /** + * Given scenario: a non-admin user requests a fix-assets job + * Expected result: SecurityException is thrown + */ + @Test(expected = SecurityException.class) + public void test_requestFixAssetsJob_asNonAdmin_throwsSecurity() { + final HttpServletRequest request = createRequestForUser(nonAdminUser); + resource.requestFixAssetsJob(request, mockResponse); + } + + /** + * Given scenario: a fix-assets job is already active and another POST arrives + * Expected result: ConflictException is thrown for the second request + */ + @Test(expected = ConflictException.class) + public void test_requestFixAssetsJob_whileActive_throwsConflict() throws Exception { + final HttpServletRequest request = createAdminRequest(); + final ResponseEntityJobStatusView first = + resource.requestFixAssetsJob(request, mockResponse); + try { + resource.requestFixAssetsJob(request, mockResponse); + } finally { + awaitJobCompletion(first.getEntity().jobId(), 60); + } + } + + // ==================== GET /assets/_fix ==================== + + /** + * Given scenario: a fix-assets job has completed, admin queries the latest + * Expected result: the most recently completed job is returned + */ + @Test + public void test_getLatestFixAssetsJob_asAdmin_returnsJob() throws Exception { + final HttpServletRequest request = createAdminRequest(); + + final ResponseEntityJobStatusView created = + resource.requestFixAssetsJob(request, mockResponse); + awaitJobCompletion(created.getEntity().jobId(), 60); + + final ResponseEntityView latest = + resource.getLatestFixAssetsJob(request, mockResponse); + assertNotNull(latest); + assertNotNull("A completed job should be returned", latest.getEntity()); + assertEquals(created.getEntity().jobId(), latest.getEntity().id()); + } + + /** + * Given scenario: a non-admin user queries the latest fix-assets job + * Expected result: SecurityException is thrown + */ + @Test(expected = SecurityException.class) + public void test_getLatestFixAssetsJob_asNonAdmin_throwsSecurity() { + final HttpServletRequest request = createRequestForUser(nonAdminUser); + resource.getLatestFixAssetsJob(request, mockResponse); + } + + // ==================== POST /assets/_clean ==================== + + /** + * Given scenario: an admin user requests a clean-assets job + * Expected result: a job is enqueued, reaches SUCCESS, and its metadata records + * {@code finalStatus=Finished} plus non-negative {@code totalFiles} + * and {@code deleted} counters + */ + @Test + public void test_requestCleanAssetsJob_asAdmin_enqueuesAndCompletes() throws Exception { + final HttpServletRequest request = createAdminRequest(); + + final ResponseEntityJobStatusView view = + resource.requestCleanAssetsJob(request, mockResponse); + + assertNotNull(view); + final JobStatusResponse status = view.getEntity(); + assertNotNull(status); + assertNotNull("jobId must be present", status.jobId()); + + final Job completed = awaitJobCompletion(status.jobId(), 120); + assertEquals("Clean-assets job should complete successfully", + JobState.SUCCESS, completed.state()); + + final Map metadata = extractMetadata(completed); + assertEquals("Clean-assets must mark the process as Finished", + "Finished", metadata.get("finalStatus")); + assertTrue("totalFiles counter must be present and non-negative", + ((Number) metadata.get("totalFiles")).intValue() >= 0); + assertTrue("deleted counter must be present and non-negative", + ((Number) metadata.get("deleted")).intValue() >= 0); + } + + /** + * Given scenario: a UUID-named directory is planted under the assets root and + * clean-assets runs (the UUID cannot match any real contentlet inode) + * Expected result: the planted directory is deleted and the {@code deleted} counter + * in the job metadata is at least 1 + */ + @Test + public void test_cleanAssets_deletesPlantedOrphanDirectory() throws Exception { + final String assetsRoot = APILocator.getFileAssetAPI().getRealAssetsRootPath(); + assertNotNull("assets root path must be resolvable", assetsRoot); + + final String fakeInode = UUIDGenerator.generateUuid(); + final File hexLevel1 = new File(assetsRoot, String.valueOf(fakeInode.charAt(0))); + final File hexLevel2 = new File(hexLevel1, String.valueOf(fakeInode.charAt(1))); + final File orphanDir = new File(hexLevel2, fakeInode); + + assertTrue(orphanDir.mkdirs()); + Files.write(new File(orphanDir, "orphan.bin").toPath(), new byte[]{0, 1, 2, 3}); + assertTrue(orphanDir.isDirectory()); + + try { + final HttpServletRequest request = createAdminRequest(); + final ResponseEntityJobStatusView view = + resource.requestCleanAssetsJob(request, mockResponse); + final Job completed = awaitJobCompletion(view.getEntity().jobId(), 120); + + assertEquals(JobState.SUCCESS, completed.state()); + assertFalse("Planted orphan directory must be deleted by clean-assets", + orphanDir.exists()); + + final Map metadata = extractMetadata(completed); + assertTrue("deleted counter should reflect at least our planted orphan", + ((Number) metadata.get("deleted")).intValue() >= 1); + } finally { + if (orphanDir.exists()) { + FileUtils.deleteQuietly(orphanDir); + } + } + } + + /** + * Given scenario: a non-admin user requests a clean-assets job + * Expected result: SecurityException is thrown + */ + @Test(expected = SecurityException.class) + public void test_requestCleanAssetsJob_asNonAdmin_throwsSecurity() { + final HttpServletRequest request = createRequestForUser(nonAdminUser); + resource.requestCleanAssetsJob(request, mockResponse); + } + + /** + * Given scenario: a clean-assets job is already active and another POST arrives + * Expected result: ConflictException is thrown for the second request + */ + @Test(expected = ConflictException.class) + public void test_requestCleanAssetsJob_whileActive_throwsConflict() throws Exception { + final HttpServletRequest request = createAdminRequest(); + final ResponseEntityJobStatusView first = + resource.requestCleanAssetsJob(request, mockResponse); + try { + resource.requestCleanAssetsJob(request, mockResponse); + } finally { + awaitJobCompletion(first.getEntity().jobId(), 120); + } + } + + // ==================== GET /assets/_clean ==================== + + /** + * Given scenario: a clean-assets job has completed, admin queries the latest + * Expected result: the most recently completed job is returned + */ + @Test + public void test_getLatestCleanAssetsJob_asAdmin_returnsJob() throws Exception { + final HttpServletRequest request = createAdminRequest(); + + final ResponseEntityJobStatusView created = + resource.requestCleanAssetsJob(request, mockResponse); + awaitJobCompletion(created.getEntity().jobId(), 120); + + final ResponseEntityView latest = + resource.getLatestCleanAssetsJob(request, mockResponse); + assertNotNull(latest); + assertNotNull("A completed job should be returned", latest.getEntity()); + assertEquals(created.getEntity().jobId(), latest.getEntity().id()); + } + + /** + * Given scenario: a non-admin user queries the latest clean-assets job + * Expected result: SecurityException is thrown + */ + @Test(expected = SecurityException.class) + public void test_getLatestCleanAssetsJob_asNonAdmin_throwsSecurity() { + final HttpServletRequest request = createRequestForUser(nonAdminUser); + resource.getLatestCleanAssetsJob(request, mockResponse); + } + // ==================== Helpers ==================== private HttpServletRequest createAdminRequest() { @@ -178,4 +425,35 @@ private static HttpServletRequest createRequestForUser(final User user) { request.setAttribute(WebKeys.USER, user); return request; } + + private static Job awaitJobCompletion(final String jobId, final int timeoutSeconds) + throws Exception { + final long deadline = System.currentTimeMillis() + timeoutSeconds * 1000L; + Job job = null; + while (System.currentTimeMillis() < deadline) { + job = jobQueueManagerAPI.getJob(jobId); + if (job != null && isTerminal(job.state())) { + return job; + } + Thread.sleep(500); + } + throw new AssertionError( + "Job " + jobId + " did not complete within " + timeoutSeconds + "s; " + + "last state=" + (job == null ? "null" : job.state())); + } + + private static boolean isTerminal(final JobState state) { + return state == JobState.SUCCESS + || state == JobState.FAILED + || state == JobState.CANCELED + || state == JobState.ABANDONED_PERMANENTLY; + } + + private static Map extractMetadata(final Job job) { + final Optional result = job.result(); + assertTrue("Completed job must carry a JobResult", result.isPresent()); + final Optional> metadata = result.get().metadata(); + assertTrue("JobResult must carry a metadata map", metadata.isPresent()); + return metadata.get(); + } }