From b9dfc8ad4bd7e5ca2b7188b616f31bdb84456da5 Mon Sep 17 00:00:00 2001 From: Jonathan Gamba Date: Mon, 30 Sep 2024 10:27:30 -0600 Subject: [PATCH] Feat (Core): PostgresJobQueue Implementation and Job Processing Enhancements (#30175) This PR introduces the `PostgresJobQueue` implementation, fulfilling the requirement for a default job queue using PostgreSQL. In the process of developing this implementation, we've made significant improvements to the overall job processing system, including enhancements to the `JobQueueManagerAPIImpl` and related components. ### Key Features of `PostgresJobQueue`: 1. **Efficient Job Management:** Implements all required operations for job queue management, including creating, retrieving, updating, and monitoring jobs. 2. **JSONB Field Usage:** Utilizes PostgreSQL's JSONB fields for storing job parameters and results, allowing for flexible and efficient data storage. 3. **Optimized Queries:** Implements efficient SQL queries, including the use of SELECT FOR UPDATE SKIP LOCKED for concurrent job processing. 4. **Pagination Support:** Provides paginated results for job listings, improving performance for large job sets. 5. **Comprehensive Job Lifecycle Management:** Handles all stages of a job's lifecycle, from creation to completion or failure. 6. **Transactional Integrity:** Ensures data consistency through proper transaction management. ### Improvements to Job Processing System: 1. **Enhanced Error Handling:** Implemented a more robust error handling mechanism, including the use of custom exceptions for different error scenarios. 2. **Improved Retry Logic:** Refined the retry mechanism to handle transient failures more gracefully. 3. **Real-Time Job Monitoring:** Enhanced the real-time monitoring capabilities, allowing for more responsive job status updates. 4. **Progress Tracking:** Improved the progress tracking system, providing more accurate and timely updates on job completion percentage. 5. **Cancellation Support:** Implemented a more reliable job cancellation process. ### `JobQueueManagerAPIImpl` Enhancements: 1. **Event-Driven Updates:** Implemented an event-driven system for job status updates, improving system responsiveness. 2. **More Granular Job State Transitions:** Implemented more detailed job state transitions, providing better insights into job processing stages. ### Testing and Documentation: Developed an extensive set of unit and integration tests for the PostgresJobQueue implementation. ## Simple class diagram with main operations and interactions ```mermaid classDiagram class JobQueueManagerAPI { <> +start() +close() +createJob(queueName: String, parameters: Map): String +getJob(jobId: String): Job +getJobs(page: int, pageSize: int): JobPaginatedResult +cancelJob(jobId: String) +watchJob(jobId: String, watcher: Consumer~Job~) +setRetryStrategy(queueName: String, retryStrategy: RetryStrategy) } class JobQueueManagerAPIImpl { -jobQueue: JobQueue -processors: Map~String, JobProcessor~ -retryStrategies: Map~String, RetryStrategy~ -circuitBreaker: CircuitBreaker +start() +close() -processJobs() -processJobWithRetry(job: Job) } class JobQueue { <> +createJob(queueName: String, parameters: Map): String +getJob(jobId: String): Job +getActiveJobs(queueName: String, page: int, pageSize: int): JobPaginatedResult +getCompletedJobs(queueName: String, startDate: LocalDateTime, endDate: LocalDateTime, page: int, pageSize: int): JobPaginatedResult +updateJobStatus(job: Job) +nextJob(): Job +updateJobProgress(jobId: String, progress: float) } class PostgresJobQueue { -objectMapper: ObjectMapper +createJob(queueName: String, parameters: Map): String +getJob(jobId: String): Job +getActiveJobs(queueName: String, page: int, pageSize: int): JobPaginatedResult +getCompletedJobs(queueName: String, startDate: LocalDateTime, endDate: LocalDateTime, page: int, pageSize: int): JobPaginatedResult +updateJobStatus(job: Job) +nextJob(): Job +updateJobProgress(jobId: String, progress: float) } class Job { +id: String +queueName: String +state: JobState +parameters: Map +progress: float +createdAt: LocalDateTime +updatedAt: LocalDateTime +startedAt: Optional~LocalDateTime~ +completedAt: Optional~LocalDateTime~ +result: Optional~JobResult~ } class JobProcessor { <> +process(job: Job) +getResultMetadata(job: Job): Map } class Cancellable { <> +cancel(job: Job) } class RetryStrategy { <> +shouldRetry(job: Job, exceptionClass: Class): boolean +nextRetryDelay(job: Job): long } class CircuitBreaker { -failureThreshold: int -resetTimeout: long +allowRequest(): boolean +recordFailure() +reset() } class RealTimeJobMonitor { -jobWatchers: Map~String, List~Consumer~Job~~~ +registerWatcher(jobId: String, watcher: Consumer~Job~) +updateWatchers(updatedJobs: List~Job~) } JobQueueManagerAPI <|.. JobQueueManagerAPIImpl JobQueueManagerAPIImpl --> JobQueue JobQueueManagerAPIImpl --> JobProcessor JobQueueManagerAPIImpl --> RetryStrategy JobQueueManagerAPIImpl --> CircuitBreaker JobQueueManagerAPIImpl --> RealTimeJobMonitor JobQueue <|.. PostgresJobQueue JobQueueManagerAPIImpl ..> Job JobProcessor <|-- Cancellable ``` --- .../jobs/business/api/JobQueueConfig.java | 16 +- .../business/api/JobQueueConfigProducer.java | 10 +- .../jobs/business/api/JobQueueManagerAPI.java | 31 +- .../business/api/JobQueueManagerAPIImpl.java | 587 +++++++++++++---- .../business/api/events/EventProducer.java | 57 ++ .../business/api/events/JobCanceledEvent.java | 38 ++ .../api/events/JobCancellingEvent.java | 38 ++ .../api/events/JobCompletedEvent.java | 38 ++ .../business/api/events/JobCreatedEvent.java | 60 ++ .../business/api/events/JobFailedEvent.java | 38 ++ .../api/events/JobProgressUpdatedEvent.java | 38 ++ .../api/events/JobRemovedFromQueueEvent.java | 38 ++ .../business/api/events/JobStartedEvent.java | 38 ++ .../api/events/RealTimeJobMonitor.java | 139 ++++ .../business/error/AbstractErrorDetail.java | 53 +- .../ExponentialBackoffRetryStrategy.java | 15 +- .../error/JobProcessorNotFoundException.java | 17 + .../error/ProcessorNotFoundException.java | 17 - .../jobs/business/error/RetryStrategy.java | 10 +- .../business/error/RetryStrategyProducer.java | 2 +- .../dotcms/jobs/business/job/AbstractJob.java | 67 +- .../job/AbstractJobPaginatedResult.java | 22 + .../jobs/business/job/AbstractJobResult.java | 24 + .../dotcms/jobs/business/job/JobResult.java | 17 - .../dotcms/jobs/business/job/JobState.java | 9 +- .../jobs/business/processor/Cancellable.java | 39 ++ .../jobs/business/processor/JobProcessor.java | 31 +- .../jobs/business/queue/DBJobTransformer.java | 267 ++++++++ .../dotcms/jobs/business/queue/JobQueue.java | 86 ++- .../jobs/business/queue/JobQueueProducer.java | 22 +- .../jobs/business/queue/PostgresJobQueue.java | 545 +++++++++++++++ .../queue/error/JobLockingException.java | 18 + .../queue/error/JobNotFoundException.java | 19 + .../queue/error/JobQueueDataException.java | 29 + .../queue/error/JobQueueException.java | 28 + .../queue/error/JobQueueFullException.java | 18 + .../com/dotmarketing/business/APILocator.java | 14 +- .../com/dotmarketing/util/ActivityLogger.java | 2 +- dotCMS/src/main/resources/postgres.sql | 49 ++ .../test/java/com/dotcms/Junit5Suite1.java | 8 +- .../api/JobQueueManagerAPICDITest.java | 19 +- .../JobQueueManagerAPIIntegrationTest.java | 549 ++++++++++++++++ .../business/api/JobQueueManagerAPITest.java | 406 ++++++++---- .../PostgresJobQueueIntegrationTest.java | 618 ++++++++++++++++++ .../util/IntegrationTestInitService.java | 28 +- 45 files changed, 3754 insertions(+), 460 deletions(-) create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/api/events/EventProducer.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCanceledEvent.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCancellingEvent.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCompletedEvent.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCreatedEvent.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobFailedEvent.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobProgressUpdatedEvent.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobRemovedFromQueueEvent.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobStartedEvent.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/api/events/RealTimeJobMonitor.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/error/JobProcessorNotFoundException.java delete mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/error/ProcessorNotFoundException.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/job/AbstractJobPaginatedResult.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/job/AbstractJobResult.java delete mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/job/JobResult.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/processor/Cancellable.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/queue/DBJobTransformer.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/queue/PostgresJobQueue.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobLockingException.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobNotFoundException.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobQueueDataException.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobQueueException.java create mode 100644 dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobQueueFullException.java create mode 100644 dotcms-integration/src/test/java/com/dotcms/jobs/business/api/JobQueueManagerAPIIntegrationTest.java create mode 100644 dotcms-integration/src/test/java/com/dotcms/jobs/business/queue/PostgresJobQueueIntegrationTest.java diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueConfig.java b/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueConfig.java index cdfe2b29d287..9bfaa637063f 100644 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueConfig.java +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueConfig.java @@ -10,13 +10,18 @@ public class JobQueueConfig { */ private final int threadPoolSize; + // The interval in milliseconds to poll for job updates + private final int pollJobUpdatesIntervalMilliseconds; + /** * Constructs a new JobQueueConfig * * @param threadPoolSize The number of threads to use for job processing. + * @param pollJobUpdatesIntervalMilliseconds The interval in milliseconds to poll for job updates. */ - public JobQueueConfig(int threadPoolSize) { + public JobQueueConfig(int threadPoolSize, int pollJobUpdatesIntervalMilliseconds) { this.threadPoolSize = threadPoolSize; + this.pollJobUpdatesIntervalMilliseconds = pollJobUpdatesIntervalMilliseconds; } /** @@ -28,4 +33,13 @@ public int getThreadPoolSize() { return threadPoolSize; } + /** + * Gets the interval in milliseconds to poll for job updates. + * + * @return The interval in milliseconds to poll for job updates. + */ + public int getPollJobUpdatesIntervalMilliseconds() { + return pollJobUpdatesIntervalMilliseconds; + } + } \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueConfigProducer.java b/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueConfigProducer.java index 4d3710bc51ff..8e10fe7e47b9 100644 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueConfigProducer.java +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueConfigProducer.java @@ -16,6 +16,11 @@ public class JobQueueConfigProducer { "JOB_QUEUE_THREAD_POOL_SIZE", 10 ); + // The interval in milliseconds to poll for job updates. + static final int DEFAULT_POLL_JOB_UPDATES_INTERVAL_MILLISECONDS = Config.getIntProperty( + "JOB_QUEUE_POLL_JOB_UPDATES_INTERVAL_MILLISECONDS", 1000 + ); + /** * Produces a JobQueueConfig object. This method is called by the CDI container to create a * JobQueueConfig instance when it is necessary for dependency injection. @@ -24,7 +29,10 @@ public class JobQueueConfigProducer { */ @Produces public JobQueueConfig produceJobQueueConfig() { - return new JobQueueConfig(DEFAULT_THREAD_POOL_SIZE); + return new JobQueueConfig( + DEFAULT_THREAD_POOL_SIZE, + DEFAULT_POLL_JOB_UPDATES_INTERVAL_MILLISECONDS + ); } } \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueManagerAPI.java b/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueManagerAPI.java index 8bd0c59ff822..aa74dd88257f 100644 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueManagerAPI.java +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueManagerAPI.java @@ -1,12 +1,13 @@ package com.dotcms.jobs.business.api; import com.dotcms.jobs.business.error.CircuitBreaker; -import com.dotcms.jobs.business.error.JobCancellationException; -import com.dotcms.jobs.business.error.ProcessorNotFoundException; +import com.dotcms.jobs.business.error.JobProcessorNotFoundException; import com.dotcms.jobs.business.error.RetryStrategy; import com.dotcms.jobs.business.job.Job; +import com.dotcms.jobs.business.job.JobPaginatedResult; import com.dotcms.jobs.business.processor.JobProcessor; -import java.util.List; +import com.dotcms.jobs.business.queue.JobQueue; +import com.dotmarketing.exception.DotDataException; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -15,7 +16,7 @@ * Defines the contract for interacting with the job queue system. This interface provides methods * for managing jobs, processors, and the overall state of the job queue. */ -public interface JobQueueManagerAPI extends AutoCloseable { +public interface JobQueueManagerAPI { /** * Starts the job queue manager, initializing the thread pool for job processing. @@ -62,35 +63,38 @@ public interface JobQueueManagerAPI extends AutoCloseable { * @param queueName The name of the queue * @param parameters The parameters for the job * @return The ID of the created job - * @throws ProcessorNotFoundException if no processor is registered for the specified queue + * @throws JobProcessorNotFoundException if no processor is registered for the specified queue + * @throws DotDataException if there's an error creating the job */ String createJob(String queueName, Map parameters) - throws ProcessorNotFoundException; + throws JobProcessorNotFoundException, DotDataException; /** * Retrieves a job by its ID. * * @param jobId The ID of the job * @return The Job object, or null if not found + * @throws DotDataException if there's an error fetching the job */ - Job getJob(String jobId); + Job getJob(String jobId) throws DotDataException; /** * Retrieves a list of jobs. * * @param page The page number * @param pageSize The number of jobs per page - * @return A list of Job objects + * @return A result object containing the list of active jobs and pagination information. + * @throws DotDataException if there's an error fetching the jobs */ - List getJobs(int page, int pageSize); + JobPaginatedResult getJobs(int page, int pageSize) throws DotDataException; /** * Cancels a job. * * @param jobId The ID of the job to cancel - * @throws JobCancellationException if the job cannot be cancelled + * @throws DotDataException if there's an error cancelling the job */ - void cancelJob(String jobId) throws JobCancellationException; + void cancelJob(String jobId) throws DotDataException; /** * Registers a watcher for a specific job. @@ -113,6 +117,11 @@ String createJob(String queueName, Map parameters) */ CircuitBreaker getCircuitBreaker(); + /** + * @return The JobQueue instance + */ + JobQueue getJobQueue(); + /** * @return The size of the thread pool */ diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueManagerAPIImpl.java b/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueManagerAPIImpl.java index afcf74f3e2d2..f675c331d21f 100644 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueManagerAPIImpl.java +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/api/JobQueueManagerAPIImpl.java @@ -1,28 +1,51 @@ package com.dotcms.jobs.business.api; +import com.dotcms.business.CloseDBIfOpened; +import com.dotcms.business.WrapInTransaction; +import com.dotcms.jobs.business.api.events.EventProducer; +import com.dotcms.jobs.business.api.events.JobCanceledEvent; +import com.dotcms.jobs.business.api.events.JobCancellingEvent; +import com.dotcms.jobs.business.api.events.JobCompletedEvent; +import com.dotcms.jobs.business.api.events.JobCreatedEvent; +import com.dotcms.jobs.business.api.events.JobFailedEvent; +import com.dotcms.jobs.business.api.events.JobProgressUpdatedEvent; +import com.dotcms.jobs.business.api.events.JobRemovedFromQueueEvent; +import com.dotcms.jobs.business.api.events.JobStartedEvent; +import com.dotcms.jobs.business.api.events.RealTimeJobMonitor; import com.dotcms.jobs.business.error.CircuitBreaker; import com.dotcms.jobs.business.error.ErrorDetail; -import com.dotcms.jobs.business.error.JobCancellationException; -import com.dotcms.jobs.business.error.ProcessorNotFoundException; +import com.dotcms.jobs.business.error.JobProcessorNotFoundException; import com.dotcms.jobs.business.error.RetryStrategy; import com.dotcms.jobs.business.job.Job; +import com.dotcms.jobs.business.job.JobPaginatedResult; +import com.dotcms.jobs.business.job.JobResult; import com.dotcms.jobs.business.job.JobState; +import com.dotcms.jobs.business.processor.Cancellable; +import com.dotcms.jobs.business.processor.DefaultProgressTracker; import com.dotcms.jobs.business.processor.JobProcessor; import com.dotcms.jobs.business.processor.ProgressTracker; import com.dotcms.jobs.business.queue.JobQueue; +import com.dotcms.jobs.business.queue.error.JobNotFoundException; +import com.dotcms.jobs.business.queue.error.JobQueueDataException; +import com.dotcms.jobs.business.queue.error.JobQueueException; +import com.dotmarketing.exception.DoesNotExistException; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotRuntimeException; import com.dotmarketing.util.Logger; import com.google.common.annotations.VisibleForTesting; import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import javax.enterprise.context.ApplicationScoped; import javax.inject.Inject; @@ -31,13 +54,11 @@ * Manages the processing of jobs in a distributed job queue system. This class is responsible for * job creation, execution, monitoring, and error handling. *
{@code
- *     public static void main(String[] args) {
  *
- *        // Create the job queue
- *        JobQueue jobQueue = new PostgresJobQueue();
+ *     @Inject
+ *     private JobQueueManagerAPI jobQueueManagerAPI;
  *
- *        // Create and start the job queue manager
- *        JobQueueManagerAPIImpl jobQueueManagerAPI = new JobQueueManagerAPIImpl(jobQueue, 5); // 5 threads
+ *     public static void main(String[] args) {
  *
  *        // (Optional) Set up a retry strategy for content import jobs, if not set, the default retry strategy will be used
  *        RetryStrategy contentImportRetryStrategy = new ExponentialBackoffRetryStrategy(
@@ -81,29 +102,57 @@ public class JobQueueManagerAPIImpl implements JobQueueManagerAPI {
     private final Map processors;
     private final int threadPoolSize;
     private ExecutorService executorService;
-    private final Map>> jobWatchers;
     private final Map retryStrategies;
     private final RetryStrategy defaultRetryStrategy;
 
+    private final ScheduledExecutorService pollJobUpdatesScheduler;
+    private LocalDateTime lastPollJobUpdateTime = LocalDateTime.now();
+
+    private final RealTimeJobMonitor realTimeJobMonitor;
+    private final EventProducer eventProducer;
+
     /**
      * Constructs a new JobQueueManagerAPIImpl.
+     * This constructor initializes the job queue manager with all necessary dependencies and configurations.
      *
-     * @param jobQueue             The JobQueue implementation to use.
-     * @param jobQueueConfig       The JobQueueConfig implementation to use.
-     * @param circuitBreaker       The CircuitBreaker implementation to use.
-     * @param defaultRetryStrategy The default retry strategy to use.
+     * @param jobQueue             The JobQueue implementation to use for managing jobs.
+     * @param jobQueueConfig       The JobQueueConfig implementation providing configuration settings.
+     * @param circuitBreaker       The CircuitBreaker implementation for fault tolerance.
+     * @param defaultRetryStrategy The default retry strategy to use for failed jobs.
+     * @param realTimeJobMonitor   The RealTimeJobMonitor for handling real-time job updates.
+     * @param eventProducer        The EventProducer for firing job-related events.
+     * 

+ * This constructor performs the following initializations: + * - Sets up the job queue and related configurations. + * - Initializes thread pool and job processors. + * - Sets up the circuit breaker and retry strategies. + * - Configures the job update polling mechanism. + * - Initializes event handlers for various job state changes. */ @Inject - public JobQueueManagerAPIImpl(JobQueue jobQueue, JobQueueConfig jobQueueConfig, - CircuitBreaker circuitBreaker, RetryStrategy defaultRetryStrategy) { + public JobQueueManagerAPIImpl(JobQueue jobQueue, + JobQueueConfig jobQueueConfig, + CircuitBreaker circuitBreaker, + RetryStrategy defaultRetryStrategy, + RealTimeJobMonitor realTimeJobMonitor, + EventProducer eventProducer) { this.jobQueue = jobQueue; this.threadPoolSize = jobQueueConfig.getThreadPoolSize(); this.processors = new ConcurrentHashMap<>(); - this.jobWatchers = new ConcurrentHashMap<>(); this.retryStrategies = new ConcurrentHashMap<>(); this.defaultRetryStrategy = defaultRetryStrategy; this.circuitBreaker = circuitBreaker; + + this.pollJobUpdatesScheduler = Executors.newSingleThreadScheduledExecutor(); + pollJobUpdatesScheduler.scheduleAtFixedRate( + this::pollJobUpdates, 0, + jobQueueConfig.getPollJobUpdatesIntervalMilliseconds(), TimeUnit.MILLISECONDS + ); + + // Events + this.realTimeJobMonitor = realTimeJobMonitor; + this.eventProducer = eventProducer; } @Override @@ -160,19 +209,11 @@ public void close() throws Exception { isShuttingDown = true; Logger.info(this, "Closing JobQueue and stopping all job processing."); - executorService.shutdownNow(); - try { - if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) { - Logger.error(this, "ExecutorService did not terminate"); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - Logger.error(this, "Interrupted while waiting for jobs to complete", e); - } finally { - isShuttingDown = false; - } + closeExecutorService(executorService); + closeExecutorService(pollJobUpdatesScheduler); + isShuttingDown = false; isClosed = true; Logger.info(this, "JobQueue has been successfully closed."); } else { @@ -187,58 +228,86 @@ public void registerProcessor(final String queueName, final JobProcessor process processors.put(queueName, processor); } + @WrapInTransaction @Override - public String createJob(final String queueName, final Map parameters) { + public String createJob(final String queueName, final Map parameters) + throws JobProcessorNotFoundException, DotDataException { if (!processors.containsKey(queueName)) { - final var error = new ProcessorNotFoundException(queueName); + final var error = new JobProcessorNotFoundException(queueName); Logger.error(JobQueueManagerAPIImpl.class, error); throw error; } - return jobQueue.addJob(queueName, parameters); + try { + String jobId = jobQueue.createJob(queueName, parameters); + eventProducer.getEvent(JobCreatedEvent.class).fire( + new JobCreatedEvent(jobId, queueName, LocalDateTime.now(), parameters) + ); + return jobId; + } catch (JobQueueException e) { + throw new DotDataException("Error creating job", e); + } } + @CloseDBIfOpened @Override - public Job getJob(final String jobId) { - return jobQueue.getJob(jobId); + public Job getJob(final String jobId) throws DotDataException { + try { + return jobQueue.getJob(jobId); + } catch (JobNotFoundException e) { + throw new DoesNotExistException(e); + } catch (JobQueueDataException e) { + throw new DotDataException("Error fetching job", e); + } } + @CloseDBIfOpened @Override - public List getJobs(final int page, final int pageSize) { - return jobQueue.getJobs(page, pageSize); + public JobPaginatedResult getJobs(final int page, final int pageSize) throws DotDataException { + try { + return jobQueue.getJobs(page, pageSize); + } catch (JobQueueDataException e) { + throw new DotDataException("Error fetching jobs", e); + } } + @WrapInTransaction @Override - public void cancelJob(final String jobId) { - - Job job = jobQueue.getJob(jobId); - if (job != null) { - - final var processor = processors.get(job.queueName()); - if (processor != null && processor.canCancel(job)) { + public void cancelJob(final String jobId) throws DotDataException { + + final Job job; + try { + job = jobQueue.getJob(jobId); + } catch (JobNotFoundException e) { + throw new DoesNotExistException(e); + } catch (JobQueueDataException e) { + throw new DotDataException("Error fetching job", e); + } - try { + final var processor = processors.get(job.queueName()); + if (processor instanceof Cancellable) { - Logger.info(this, "Cancelling job " + jobId); + try { - processor.cancel(job); + Logger.info(this, "Cancelling job " + jobId); - Job cancelledJob = job.withState(JobState.CANCELLED); - jobQueue.updateJobStatus(cancelledJob); - notifyJobWatchers(cancelledJob); - } catch (Exception e) { - final var error = new JobCancellationException(jobId, e.getMessage()); - Logger.error(JobQueueManagerAPIImpl.class, error); - throw error; - } - } else { - final var error = new JobCancellationException(jobId, "Job cannot be cancelled"); + ((Cancellable) processor).cancel(job); + handleJobCancelling(job, processor); + } catch (Exception e) { + final var error = new DotDataException("Error cancelling job " + jobId, e); Logger.error(JobQueueManagerAPIImpl.class, error); throw error; } } else { - final var error = new JobCancellationException(jobId, "Job not found"); + + if (processor == null) { + final var error = new JobProcessorNotFoundException(job.queueName()); + Logger.error(JobQueueManagerAPIImpl.class, error); + throw error; + } + + final var error = new DotDataException(jobId, "Job " + jobId + " cannot be canceled"); Logger.error(JobQueueManagerAPIImpl.class, error); throw error; } @@ -246,9 +315,7 @@ public void cancelJob(final String jobId) { @Override public void watchJob(final String jobId, final Consumer watcher) { - jobWatchers.computeIfAbsent(jobId, k -> new CopyOnWriteArrayList<>()).add(watcher); - Job currentJob = jobQueue.getJob(jobId); - watcher.accept(currentJob); + realTimeJobMonitor.registerWatcher(jobId, watcher); } @Override @@ -262,6 +329,12 @@ public CircuitBreaker getCircuitBreaker() { return this.circuitBreaker; } + @Override + @VisibleForTesting + public JobQueue getJobQueue() { + return this.jobQueue; + } + @Override @VisibleForTesting public int getThreadPoolSize() { @@ -275,19 +348,26 @@ public RetryStrategy getDefaultRetryStrategy() { } /** - * Notifies all registered watchers for a job of its current state. - * - * @param job The job whose watchers to notify. + * Polls the job queue for updates to watched jobs and notifies their watchers. */ - private void notifyJobWatchers(final Job job) { - List> watchers = jobWatchers.get(job.id()); - if (watchers != null) { - watchers.forEach(watcher -> watcher.accept(job)); - if (job.state() == JobState.COMPLETED - || job.state() == JobState.FAILED - || job.state() == JobState.CANCELLED) { - jobWatchers.remove(job.id()); + @CloseDBIfOpened + private void pollJobUpdates() { + + try { + final var watchedJobIds = realTimeJobMonitor.getWatchedJobIds(); + if (watchedJobIds.isEmpty()) { + return; // No jobs are being watched, skip polling } + + final var currentPollTime = LocalDateTime.now(); + List updatedJobs = jobQueue.getUpdatedJobsSince( + watchedJobIds, lastPollJobUpdateTime + ); + realTimeJobMonitor.updateWatchers(updatedJobs); + lastPollJobUpdateTime = currentPollTime; + } catch (JobQueueDataException e) { + Logger.error(this, "Error polling job updates: " + e.getMessage(), e); + throw new DotRuntimeException("Error polling job updates", e); } } @@ -296,16 +376,36 @@ private void notifyJobWatchers(final Job job) { * * @param job The job whose progress to update. * @param progressTracker The processor progress tracker + * @param previousProgress The previous progress value */ - private void updateJobProgress(final Job job, final ProgressTracker progressTracker) { - if (job != null) { + @WrapInTransaction + private float updateJobProgress(final Job job, final ProgressTracker progressTracker, + final float previousProgress) throws DotDataException { + + try { + if (job != null) { + + float progress = progressTracker.progress(); + + // Only update progress if it has changed + if (progress > previousProgress) { + + Job updatedJob = job.withProgress(progress); - float progress = progressTracker.progress(); - Job updatedJob = job.withProgress(progress); + jobQueue.updateJobProgress(job.id(), updatedJob.progress()); + eventProducer.getEvent(JobProgressUpdatedEvent.class).fire( + new JobProgressUpdatedEvent(updatedJob, LocalDateTime.now()) + ); - jobQueue.updateJobProgress(job.id(), updatedJob.progress()); - notifyJobWatchers(updatedJob); + return progress; + } + } + } catch (JobQueueDataException e) { + Logger.error(this, "Error updating job progress: " + e.getMessage(), e); + throw new DotDataException("Error updating job progress", e); } + + return -1; } /** @@ -323,9 +423,8 @@ private void processJobs() { try { - Job job = jobQueue.nextJob(); - if (job != null) { - processJobWithRetry(job); + boolean jobProcessed = processNextJob(); + if (jobProcessed) { emptyQueueCount = 0; } else { // If no jobs were found, wait for a short time before checking again @@ -344,6 +443,48 @@ private void processJobs() { } } + /** + * Processes the next job in the queue. + * + * @return {@code true} if a job was processed, {@code false} if the queue is empty. + */ + @CloseDBIfOpened + private boolean processNextJob() throws DotDataException { + + try { + Job job = jobQueue.nextJob(); + if (job != null) { + processJobWithRetry(job); + return true; + } + + return false; + } catch (JobQueueException e) { + Logger.error(this, "Error fetching next job: " + e.getMessage(), e); + throw new DotDataException("Error fetching next job", e); + } + } + + /** + * Closes an ExecutorService, shutting it down and waiting for any running jobs to complete. + * + * @param executor The ExecutorService to close. + */ + private void closeExecutorService(final ExecutorService executor) { + + executor.shutdown(); + + try { + if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { + executor.shutdownNow(); + } + } catch (InterruptedException e) { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + Logger.error(this, "Interrupted while waiting for jobs to complete", e); + } + } + /** * Checks if the circuit breaker is open and handles the waiting period if it is. * @@ -373,7 +514,7 @@ private boolean isCircuitBreakerOpen() { * * @param job The job to be processed. */ - private void processJobWithRetry(final Job job) { + private void processJobWithRetry(final Job job) throws DotDataException { if (job.state() == JobState.FAILED) { @@ -386,8 +527,12 @@ private void processJobWithRetry(final Job job) { Logger.debug(this, "Job " + job.id() + " is not ready for retry, " + "putting back in queue."); - // Put the job back in the queue for later retry - jobQueue.putJobBackInQueue(job); + try { + // Put the job back in the queue for later retry + jobQueue.putJobBackInQueue(job); + } catch (JobQueueDataException e) { + throw new DotDataException("Error re-queueing job", e); + } } } else { handleNonRetryableFailedJob(job); @@ -402,9 +547,13 @@ private void processJobWithRetry(final Job job) { * @param job The job to check for retry eligibility. * @return {@code true} if the job is ready for retry, {@code false} otherwise. */ - private boolean isReadyForRetry(Job job) { + private boolean isReadyForRetry(Job job) throws DotDataException { long now = System.currentTimeMillis(); - long nextRetryTime = job.lastRetryTimestamp() + nextRetryDelay(job); + long lastCompletedAt = job.completedAt() + .orElseThrow(() -> new DotDataException("Job has not completed at")) + .atZone(ZoneId.systemDefault()).toInstant() + .toEpochMilli(); + long nextRetryTime = lastCompletedAt + nextRetryDelay(job); return now >= nextRetryTime; } @@ -414,10 +563,18 @@ private boolean isReadyForRetry(Job job) { * * @param job The failed job that cannot be retried. */ - private void handleNonRetryableFailedJob(final Job job) { + private void handleNonRetryableFailedJob(final Job job) throws DotDataException { Logger.warn(this, "Job " + job.id() + " has failed and cannot be retried."); - jobQueue.removeJob(job.id()); + + try { + jobQueue.removeJobFromQueue(job.id()); + eventProducer.getEvent(JobRemovedFromQueueEvent.class).fire( + new JobRemovedFromQueueEvent(job, LocalDateTime.now()) + ); + } catch (JobQueueDataException e) { + throw new DotDataException("Error removing failed job", e); + } } /** @@ -425,77 +582,202 @@ private void handleNonRetryableFailedJob(final Job job) { * * @param job The job to process. */ - private void processJob(final Job job) { + private void processJob(final Job job) throws DotDataException { JobProcessor processor = processors.get(job.queueName()); if (processor != null) { - Job runningJob = job.withState(JobState.RUNNING); - jobQueue.updateJobStatus(runningJob); - notifyJobWatchers(runningJob); + final ProgressTracker progressTracker = new DefaultProgressTracker(); + Job runningJob = job.markAsRunning().withProgressTracker(progressTracker); + updateJobStatus(runningJob); + eventProducer.getEvent(JobStartedEvent.class).fire( + new JobStartedEvent(runningJob, LocalDateTime.now()) + ); try (final CloseableScheduledExecutor closeableExecutor = new CloseableScheduledExecutor()) { - final ProgressTracker progressTracker = processor.progressTracker(runningJob); - // Start a separate thread to periodically update and persist progress + AtomicReference currentProgress = new AtomicReference<>((float) 0); ScheduledExecutorService progressUpdater = closeableExecutor.getExecutorService(); progressUpdater.scheduleAtFixedRate(() -> - updateJobProgress(runningJob, progressTracker), 0, 1, TimeUnit.SECONDS + { + try { + final var progress = updateJobProgress( + runningJob, progressTracker, currentProgress.get() + ); + if (progress >= 0) { + currentProgress.set(progress); + } + } catch (DotDataException e) { + throw new DotRuntimeException("Error updating job progress", e); + } + }, 0, 1, TimeUnit.SECONDS ); - try { - processor.process(runningJob); - } finally { - // Ensure final progress is updated - updateJobProgress(runningJob, progressTracker); - } - - Job completedJob = runningJob.markAsCompleted(); - jobQueue.updateJobStatus(completedJob); + // Process the job + processor.process(runningJob); - notifyJobWatchers(completedJob); + if (jobQueue.hasJobBeenInState(runningJob.id(), JobState.CANCELLING)) { + handleJobCancellation(runningJob, processor); + } else { + handleJobCompletion(runningJob, processor); + } } catch (Exception e) { Logger.error(this, - "Error processing job " + runningJob.id() + ": " + e.getMessage(), e); - final var errorDetail = ErrorDetail.builder() - .message("Job processing failed") - .exception(e) - .exceptionClass(e.getClass().getName()) - .processingStage("Job execution") - .timestamp(LocalDateTime.now()) - .build(); - handleJobFailure(runningJob, errorDetail); + "Error processing job " + runningJob.id() + ": " + e.getMessage(), e + ); + handleJobFailure( + runningJob, processor, e, e.getMessage(), "Job execution" + ); } } else { Logger.error(this, "No processor found for queue: " + job.queueName()); - final var errorDetail = ErrorDetail.builder() - .message("No processor found for queue") - .processingStage("Processor selection") - .timestamp(LocalDateTime.now()) - .build(); - handleJobFailure(job, errorDetail); + handleJobFailure(job, null, new JobProcessorNotFoundException(job.queueName()), + "No processor found for queue", "Processor selection" + ); + } + } + + /** + * Handles the completion of a job. + * + * @param job The job that completed. + * @param processor The processor that handled the job. + */ + @WrapInTransaction + private void handleJobCompletion(final Job job, final JobProcessor processor) + throws DotDataException { + + final var resultMetadata = processor.getResultMetadata(job); + + JobResult jobResult = null; + if (resultMetadata != null && !resultMetadata.isEmpty()) { + jobResult = JobResult.builder().metadata(resultMetadata).build(); + } + + final float progress = getJobProgress(job); + + final Job completedJob = job.markAsCompleted(jobResult).withProgress(progress); + updateJobStatus(completedJob); + eventProducer.getEvent(JobCompletedEvent.class).fire( + new JobCompletedEvent(completedJob, LocalDateTime.now()) + ); + } + + /** + * Handles the cancellation of a job. + * + * @param job The job that was canceled. + * @param processor The processor that handled the job. + */ + @WrapInTransaction + private void handleJobCancelling(final Job job, final JobProcessor processor) + throws DotDataException { + + Job cancelJob = job.withState(JobState.CANCELLING); + updateJobStatus(cancelJob); + eventProducer.getEvent(JobCancellingEvent.class).fire( + new JobCancellingEvent(cancelJob, LocalDateTime.now()) + ); + } + + /** + * Handles the cancellation of a job. + * + * @param job The job that was canceled. + * @param processor The processor that handled the job. + */ + @WrapInTransaction + private void handleJobCancellation(final Job job, final JobProcessor processor) + throws DotDataException { + + final var resultMetadata = processor.getResultMetadata(job); + + JobResult jobResult = null; + if (resultMetadata != null && !resultMetadata.isEmpty()) { + jobResult = JobResult.builder().metadata(resultMetadata).build(); } + + final float progress = getJobProgress(job); + + Job canceledJob = job.markAsCanceled(jobResult).withProgress(progress); + updateJobStatus(canceledJob); + eventProducer.getEvent(JobCanceledEvent.class).fire( + new JobCanceledEvent(canceledJob, LocalDateTime.now()) + ); } /** * Handles the failure of a job * - * @param job The job that failed. - * @param errorDetail The details of the error that caused the failure. + * @param job The job that failed. + * @param processor The processor that handled the job. + * @param exception The exception that caused the failure. + * @param errorMessage The error message to include in the job result. + * @param processingStage The stage of processing where the failure occurred. */ - private void handleJobFailure(final Job job, final ErrorDetail errorDetail) { + @WrapInTransaction + private void handleJobFailure(final Job job, final JobProcessor processor, + final Exception exception, final String errorMessage, final String processingStage) + throws DotDataException { + + if (exception == null) { + throw new IllegalArgumentException("Exception cannot be null"); + } + + final var errorDetail = ErrorDetail.builder() + .message(errorMessage) + .stackTrace(stackTrace(exception)) + .exceptionClass(exception.getClass().getName()) + .processingStage(processingStage) + .timestamp(LocalDateTime.now()) + .build(); + + JobResult jobResult = JobResult.builder().errorDetail(errorDetail).build(); + + if (processor != null) { + final var resultMetadata = processor.getResultMetadata(job); + if (resultMetadata != null && !resultMetadata.isEmpty()) { + jobResult = jobResult.withMetadata(resultMetadata); + } + } + + final float progress = getJobProgress(job); - final Job failedJob = job.markAsFailed(errorDetail); - jobQueue.updateJobStatus(failedJob); - notifyJobWatchers(failedJob); + final Job failedJob = job.markAsFailed(jobResult).withProgress(progress); + updateJobStatus(failedJob); + eventProducer.getEvent(JobFailedEvent.class).fire( + new JobFailedEvent(failedJob, LocalDateTime.now()) + ); + + try { + // Put the job back in the queue for later retry + jobQueue.putJobBackInQueue(job); + } catch (JobQueueDataException e) { + throw new DotDataException("Error re-queueing job", e); + } // Record the failure in the circuit breaker getCircuitBreaker().recordFailure(); } + /** + * Updates the status of a job in the job queue. + * + * @param job The job to update. + * @throws DotDataException if there's an error updating the job status. + */ + @WrapInTransaction + private void updateJobStatus(final Job job) throws DotDataException { + try { + jobQueue.updateJobStatus(job); + } catch (JobQueueDataException e) { + throw new DotDataException("Error updating job status", e); + } + } + /** * Gets the retry strategy for a specific queue. * @@ -513,8 +795,30 @@ private RetryStrategy retryStrategy(final String queueName) { * @return {@code true} if the job is eligible for retry, {@code false} otherwise. */ private boolean canRetry(final Job job) { + final RetryStrategy retryStrategy = retryStrategy(job.queueName()); - return retryStrategy.shouldRetry(job, job.lastException().orElse(null)); + + final var jobResult = job.result(); + + Class lastExceptionClass = null; + if (jobResult.isPresent()) { + + final var errorDetailOptional = jobResult.get().errorDetail(); + + if (errorDetailOptional.isPresent()) { + final var errorDetail = errorDetailOptional.get(); + final var exceptionClass = errorDetail.exceptionClass(); + if (exceptionClass != null) { + try { + lastExceptionClass = Class.forName(errorDetail.exceptionClass()); + } catch (ClassNotFoundException e) { + Logger.error(this, "Error loading exception class: " + e.getMessage(), e); + } + } + } + } + + return retryStrategy.shouldRetry(job, (Class) lastExceptionClass); } /** @@ -528,6 +832,41 @@ private long nextRetryDelay(final Job job) { return retryStrategy.nextRetryDelay(job); } + /** + * Generates and returns the stack trace of the exception as a string. This is a derived value + * and will be computed only when accessed. + * + * @param exception The exception for which to generate the stack trace. + * @return A string representation of the exception's stack trace, or null if no exception is + * present. + */ + private String stackTrace(final Throwable exception) { + if (exception != null) { + return Arrays.stream(exception.getStackTrace()) + .map(StackTraceElement::toString) + .reduce((a, b) -> a + "\n" + b) + .orElse(""); + } + return null; + } + + /** + * Gets the progress of a job, or the progress of the job's progress tracker if present. + * + * @param job The job to get the progress for. + * @return The progress of the job, or the progress of the job's progress tracker if present. + */ + private float getJobProgress(final Job job) { + + float progress = job.progress(); + var progressTracker = job.progressTracker(); + if (progressTracker.isPresent()) { + progress = progressTracker.get().progress(); + } + + return progress; + } + /** * A wrapper class that makes ScheduledExecutorService auto-closeable. This class is designed to * be used with try-with-resources to ensure that the ScheduledExecutorService is properly shut diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/EventProducer.java b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/EventProducer.java new file mode 100644 index 000000000000..f6f22e3570f5 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/EventProducer.java @@ -0,0 +1,57 @@ +package com.dotcms.jobs.business.api.events; + +import java.lang.annotation.Annotation; +import javax.enterprise.context.ApplicationScoped; +import javax.enterprise.event.Event; +import javax.enterprise.inject.spi.BeanManager; +import javax.inject.Inject; + +/** + * A producer class for CDI events. This class provides a centralized way to obtain event objects + * for firing events within the application. + * + *

This class is application scoped, ensuring a single instance is used throughout + * the application's lifecycle.

+ */ +@ApplicationScoped +public class EventProducer { + + private BeanManager beanManager; + + public EventProducer() { + // Default constructor for CDI + } + + /** + * Constructs a new EventProducer. + * + * @param beanManager The CDI BeanManager, injected by the container. + */ + @Inject + public EventProducer(BeanManager beanManager) { + this.beanManager = beanManager; + } + + /** + * Retrieves an Event object for the specified event type and qualifiers. + * + *

This method allows for type-safe event firing. It uses the BeanManager to + * create an Event object that can be used to fire events of the specified type.

+ * + *

Usage example:

+ *
+     * EventProducer producer = ...;
+     * Event event = producer.getEvent(MyCustomEvent.class);
+     * event.fire(new MyCustomEvent(...));
+     * 
+ * + * @param The type of the event. + * @param eventType The Class object representing the event type. + * @param qualifiers Optional qualifiers for the event. + * @return An Event object that can be used to fire events of the specified type. + */ + public Event getEvent(Class eventType, Annotation... qualifiers) { + return beanManager.getEvent().select(eventType, qualifiers); + } + +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCanceledEvent.java b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCanceledEvent.java new file mode 100644 index 000000000000..9a9896604aa6 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCanceledEvent.java @@ -0,0 +1,38 @@ +package com.dotcms.jobs.business.api.events; + +import com.dotcms.jobs.business.job.Job; +import java.time.LocalDateTime; + +/** + * Event fired when a job is canceled. + */ +public class JobCanceledEvent { + + private final Job job; + private final LocalDateTime canceledAt; + + /** + * Constructs a new JobCanceledEvent. + * + * @param job The canceled job. + * @param canceledAt The timestamp when the job was canceled. + */ + public JobCanceledEvent(Job job, LocalDateTime canceledAt) { + this.job = job; + this.canceledAt = canceledAt; + } + + /** + * @return The canceled job. + */ + public Job getJob() { + return job; + } + + /** + * @return The timestamp when the job was canceled. + */ + public LocalDateTime getCanceledAt() { + return canceledAt; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCancellingEvent.java b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCancellingEvent.java new file mode 100644 index 000000000000..20877a890387 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCancellingEvent.java @@ -0,0 +1,38 @@ +package com.dotcms.jobs.business.api.events; + +import com.dotcms.jobs.business.job.Job; +import java.time.LocalDateTime; + +/** + * Event fired when a job is being canceled. + */ +public class JobCancellingEvent { + + private final Job job; + private final LocalDateTime canceledAt; + + /** + * Constructs a new JobCancellingEvent. + * + * @param job The canceled job. + * @param canceledAt The timestamp when the job was canceled. + */ + public JobCancellingEvent(Job job, LocalDateTime canceledAt) { + this.job = job; + this.canceledAt = canceledAt; + } + + /** + * @return The canceled job. + */ + public Job getJob() { + return job; + } + + /** + * @return The timestamp when the job was canceled. + */ + public LocalDateTime getCanceledAt() { + return canceledAt; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCompletedEvent.java b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCompletedEvent.java new file mode 100644 index 000000000000..7c86c3940d6d --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCompletedEvent.java @@ -0,0 +1,38 @@ +package com.dotcms.jobs.business.api.events; + +import com.dotcms.jobs.business.job.Job; +import java.time.LocalDateTime; + +/** + * Event fired when a job completes successfully. + */ +public class JobCompletedEvent { + + private final Job job; + private final LocalDateTime completedAt; + + /** + * Constructs a new JobCompletedEvent. + * + * @param job The completed job. + * @param completedAt The timestamp when the job completed. + */ + public JobCompletedEvent(Job job, LocalDateTime completedAt) { + this.job = job; + this.completedAt = completedAt; + } + + /** + * @return The completed job. + */ + public Job getJob() { + return job; + } + + /** + * @return The timestamp when the job completed. + */ + public LocalDateTime getCompletedAt() { + return completedAt; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCreatedEvent.java b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCreatedEvent.java new file mode 100644 index 000000000000..d7da52d2f5f8 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobCreatedEvent.java @@ -0,0 +1,60 @@ +package com.dotcms.jobs.business.api.events; + +import java.time.LocalDateTime; +import java.util.Map; + +/** + * Event fired when a new job is created and added to the queue. + */ +public class JobCreatedEvent { + + private final String jobId; + private final String queueName; + private final LocalDateTime createdAt; + private final Map parameters; + + /** + * Constructs a new JobCreatedEvent. + * + * @param jobId The unique identifier of the created job. + * @param queueName The name of the queue the job was added to. + * @param createdAt The timestamp when the job was created. + * @param parameters The parameters of the job. + */ + public JobCreatedEvent(String jobId, String queueName, LocalDateTime createdAt, + Map parameters) { + this.jobId = jobId; + this.queueName = queueName; + this.createdAt = createdAt; + this.parameters = parameters; + } + + /** + * @return The unique identifier of the created job. + */ + public String getJobId() { + return jobId; + } + + /** + * @return The name of the queue the job was added to. + */ + public String getQueueName() { + return queueName; + } + + /** + * @return The timestamp when the job was created. + */ + public LocalDateTime getCreatedAt() { + return createdAt; + } + + /** + * @return The parameters of the job. + */ + public Map getParameters() { + return parameters; + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobFailedEvent.java b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobFailedEvent.java new file mode 100644 index 000000000000..8d098c1eae43 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobFailedEvent.java @@ -0,0 +1,38 @@ +package com.dotcms.jobs.business.api.events; + +import com.dotcms.jobs.business.job.Job; +import java.time.LocalDateTime; + +/** + * Event fired when a job fails during processing. + */ +public class JobFailedEvent { + + private final Job job; + private final LocalDateTime failedAt; + + /** + * Constructs a new JobFailedEvent. + * + * @param job The failed job. + * @param failedAt The timestamp when the job failed. + */ + public JobFailedEvent(Job job, LocalDateTime failedAt) { + this.job = job; + this.failedAt = failedAt; + } + + /** + * @return The failed job. + */ + public Job getJob() { + return job; + } + + /** + * @return The timestamp when the job failed. + */ + public LocalDateTime getFailedAt() { + return failedAt; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobProgressUpdatedEvent.java b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobProgressUpdatedEvent.java new file mode 100644 index 000000000000..e859284ac09f --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobProgressUpdatedEvent.java @@ -0,0 +1,38 @@ +package com.dotcms.jobs.business.api.events; + +import com.dotcms.jobs.business.job.Job; +import java.time.LocalDateTime; + +/** + * Event fired when a job's progress is updated. + */ +public class JobProgressUpdatedEvent { + + private final Job job; + private final LocalDateTime updatedAt; + + /** + * Constructs a new JobProgressUpdatedEvent. + * + * @param job The job. + * @param updatedAt The timestamp when the progress was updated. + */ + public JobProgressUpdatedEvent(Job job, LocalDateTime updatedAt) { + this.job = job; + this.updatedAt = updatedAt; + } + + /** + * @return The job. + */ + public Job getJob() { + return job; + } + + /** + * @return The timestamp when the progress was updated. + */ + public LocalDateTime getUpdatedAt() { + return updatedAt; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobRemovedFromQueueEvent.java b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobRemovedFromQueueEvent.java new file mode 100644 index 000000000000..0aa08dcaa886 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobRemovedFromQueueEvent.java @@ -0,0 +1,38 @@ +package com.dotcms.jobs.business.api.events; + +import com.dotcms.jobs.business.job.Job; +import java.time.LocalDateTime; + +/** + * Event fired when a job is removed from the queue because failed and is not retryable. + */ +public class JobRemovedFromQueueEvent { + + private final Job job; + private final LocalDateTime removedAt; + + /** + * Constructs a new JobRemovedFromQueueEvent. + * + * @param job The non-retryable job. + * @param canceledAt The timestamp when the job was removed from the queue. + */ + public JobRemovedFromQueueEvent(Job job, LocalDateTime canceledAt) { + this.job = job; + this.removedAt = canceledAt; + } + + /** + * @return The non-retryable job. + */ + public Job getJob() { + return job; + } + + /** + * @return The timestamp when the job removed from the queue. + */ + public LocalDateTime getRemovedAt() { + return removedAt; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobStartedEvent.java b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobStartedEvent.java new file mode 100644 index 000000000000..dc357c6822d6 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/JobStartedEvent.java @@ -0,0 +1,38 @@ +package com.dotcms.jobs.business.api.events; + +import com.dotcms.jobs.business.job.Job; +import java.time.LocalDateTime; + +/** + * Event fired when a job starts processing. + */ +public class JobStartedEvent { + + private final Job job; + private final LocalDateTime startedAt; + + /** + * Constructs a new JobStartedEvent. + * + * @param job The started job. + * @param startedAt The timestamp when the job started processing. + */ + public JobStartedEvent(Job job, LocalDateTime startedAt) { + this.job = job; + this.startedAt = startedAt; + } + + /** + * @return The started job. + */ + public Job getJob() { + return job; + } + + /** + * @return The timestamp when the job started processing. + */ + public LocalDateTime getStartedAt() { + return startedAt; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/RealTimeJobMonitor.java b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/RealTimeJobMonitor.java new file mode 100644 index 000000000000..c55466f1f62a --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/api/events/RealTimeJobMonitor.java @@ -0,0 +1,139 @@ +package com.dotcms.jobs.business.api.events; + +import com.dotcms.jobs.business.job.Job; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Consumer; +import javax.enterprise.context.ApplicationScoped; +import javax.enterprise.event.Observes; + +/** + * Manages real-time monitoring of jobs in the system. This class handles registration of job + * watchers, updates watchers on job changes, and processes various job-related events. + */ +@ApplicationScoped +public class RealTimeJobMonitor { + + private final Map>> jobWatchers = new ConcurrentHashMap<>(); + + /** + * Registers a watcher for a specific job. + * + * @param jobId The ID of the job to watch. + * @param watcher The consumer to be notified of job updates. + */ + public void registerWatcher(String jobId, Consumer watcher) { + jobWatchers.computeIfAbsent(jobId, k -> new CopyOnWriteArrayList<>()).add(watcher); + } + + /** + * Retrieves the set of job IDs currently being watched. + * + * @return A set of job IDs. + */ + public Set getWatchedJobIds() { + return jobWatchers.keySet(); + } + + /** + * Updates watchers for a list of jobs. + * + * @param updatedJobs List of jobs that have been updated. + */ + public void updateWatchers(List updatedJobs) { + for (Job job : updatedJobs) { + updateWatchers(job); + } + } + + /** + * Updates watchers for a single job. Removes watchers if the job has reached a final state. + * + * @param job The job that has been updated. + */ + private void updateWatchers(Job job) { + + List> watchers = jobWatchers.get(job.id()); + if (watchers != null) { + watchers.forEach(watcher -> watcher.accept(job)); + } + } + + /** + * Removes the watcher associated with the specified job ID. + * + * @param jobId The ID of the job whose watcher is to be removed. + */ + private void removeWatcher(String jobId) { + jobWatchers.remove(jobId); + } + + /** + * Handles the job started event. + * + * @param event The JobStartedEvent. + */ + public void onJobStarted(@Observes JobStartedEvent event) { + updateWatchers(event.getJob()); + } + + /** + * Handles the job cancelling event. + * + * @param event The JobCancellingEvent. + */ + public void onJobCancelling(@Observes JobCancellingEvent event) { + updateWatchers(event.getJob()); + } + + /** + * Handles the job-canceled event. + * + * @param event The JobCanceledEvent. + */ + public void onJobCanceled(@Observes JobCanceledEvent event) { + updateWatchers(event.getJob()); + removeWatcher(event.getJob().id()); + } + + /** + * Handles the job completed event. + * + * @param event The JobCompletedEvent. + */ + public void onJobCompleted(@Observes JobCompletedEvent event) { + updateWatchers(event.getJob()); + removeWatcher(event.getJob().id()); + } + + /** + * Handles the job removed from queue event when failed and is not retryable. + * + * @param event The JobRemovedFromQueueEvent. + */ + public void onJobRemovedFromQueueEvent(@Observes JobRemovedFromQueueEvent event) { + removeWatcher(event.getJob().id()); + } + + /** + * Handles the job failed event. + * + * @param event The JobFailedEvent. + */ + public void onJobFailed(@Observes JobFailedEvent event) { + updateWatchers(event.getJob()); + } + + /** + * Handles the job progress updated event. + * + * @param event The JobProgressUpdatedEvent. + */ + public void onJobProgressUpdated(@Observes JobProgressUpdatedEvent event) { + updateWatchers(event.getJob()); + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/error/AbstractErrorDetail.java b/dotCMS/src/main/java/com/dotcms/jobs/business/error/AbstractErrorDetail.java index d83461487e2f..f1dfa9554684 100644 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/error/AbstractErrorDetail.java +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/error/AbstractErrorDetail.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.time.LocalDateTime; -import java.util.Arrays; import org.immutables.value.Value; /** @@ -33,6 +32,13 @@ public interface AbstractErrorDetail { */ String exceptionClass(); + /** + * Returns the stack trace of the exception as a string. + * + * @return A string representation of the exception's stack trace. + */ + String stackTrace(); + /** * Returns the timestamp when the error occurred. * @@ -47,49 +53,4 @@ public interface AbstractErrorDetail { */ String processingStage(); - /** - * Returns the original Throwable object that caused the error. - * - * @return The Throwable object, or null if no exception is available. - */ - Throwable exception(); - - /** - * Generates and returns the stack trace of the exception as a string. This is a derived value - * and will be computed only when accessed. - * - * @return A string representation of the exception's stack trace, or null if no exception is - * present. - */ - @Value.Derived - default String stackTrace() { - Throwable ex = exception(); - if (ex != null) { - return Arrays.stream(ex.getStackTrace()) - .map(StackTraceElement::toString) - .reduce((a, b) -> a + "\n" + b) - .orElse(""); - } - return null; - } - - /** - * Returns a truncated version of the stack trace. - * - * @param maxLines The maximum number of lines to include in the truncated stack trace. - * @return A string containing the truncated stacktrace, or null if no exception is present. - */ - @Value.Derived - default String truncatedStackTrace(int maxLines) { - String fullTrace = stackTrace(); - if (fullTrace == null) { - return null; - } - String[] lines = fullTrace.split("\n"); - return Arrays.stream(lines) - .limit(maxLines) - .reduce((a, b) -> a + "\n" + b) - .orElse(""); - } - } \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/error/ExponentialBackoffRetryStrategy.java b/dotCMS/src/main/java/com/dotcms/jobs/business/error/ExponentialBackoffRetryStrategy.java index ebf84e28c540..8b16953d763f 100644 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/error/ExponentialBackoffRetryStrategy.java +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/error/ExponentialBackoffRetryStrategy.java @@ -45,7 +45,7 @@ public ExponentialBackoffRetryStrategy(long initialDelay, long maxDelay, double public ExponentialBackoffRetryStrategy(long initialDelay, long maxDelay, double backoffFactor, int maxRetries, Set> retryableExceptions) { - if (initialDelay <= 0 || maxDelay <= 0 || backoffFactor <= 1 || maxRetries <= 0) { + if (initialDelay <= 0 || maxDelay <= 0 || backoffFactor <= 1) { throw new IllegalArgumentException("Invalid retry strategy parameters"); } @@ -60,12 +60,12 @@ public ExponentialBackoffRetryStrategy(long initialDelay, long maxDelay, double * Determines whether a job should be retried based on the provided job and exception. * * @param job The job in question. - * @param exception The exception that occurred during the execution of the job. + * @param exceptionClass The class of the exception that caused the failure. * @return true if the job should be retried, false otherwise. */ @Override - public boolean shouldRetry(final Job job, final Throwable exception) { - return job.retryCount() < maxRetries && isRetryableException(exception); + public boolean shouldRetry(final Job job, final Class exceptionClass) { + return job.retryCount() < maxRetries && isRetryableException(exceptionClass); } /** @@ -93,14 +93,15 @@ public int maxRetries() { } @Override - public boolean isRetryableException(final Throwable exception) { - if (exception == null) { + public boolean isRetryableException(final Class exceptionClass) { + if (exceptionClass == null) { return false; } if (retryableExceptions.isEmpty()) { return true; // If no specific exceptions are set, all are retryable } - return retryableExceptions.stream().anyMatch(clazz -> clazz.isInstance(exception)); + return retryableExceptions.stream() + .anyMatch(clazz -> clazz.isAssignableFrom(exceptionClass)); } @Override diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/error/JobProcessorNotFoundException.java b/dotCMS/src/main/java/com/dotcms/jobs/business/error/JobProcessorNotFoundException.java new file mode 100644 index 000000000000..c1b51d621bd1 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/error/JobProcessorNotFoundException.java @@ -0,0 +1,17 @@ +package com.dotcms.jobs.business.error; + +/** + * Exception thrown when no job processor is found for a specified queue. This typically occurs when + * attempting to process a job from a queue that has no registered processor. + */ +public class JobProcessorNotFoundException extends RuntimeException { + + /** + * Constructs a new JobProcessorNotFoundException with the specified queue name. + * + * @param queueName The name of the queue for which no processor was found + */ + public JobProcessorNotFoundException(String queueName) { + super("No job processor found for queue: " + queueName); + } +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/error/ProcessorNotFoundException.java b/dotCMS/src/main/java/com/dotcms/jobs/business/error/ProcessorNotFoundException.java deleted file mode 100644 index 3067eb154662..000000000000 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/error/ProcessorNotFoundException.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.dotcms.jobs.business.error; - -/** - * Exception thrown when no processor is found for a specified queue. This typically occurs when - * attempting to process a job from a queue that has no registered processor. - */ -public class ProcessorNotFoundException extends RuntimeException { - - /** - * Constructs a new NoProcessorFoundException with the specified queue name. - * - * @param queueName The name of the queue for which no processor was found - */ - public ProcessorNotFoundException(String queueName) { - super("No processor found for queue: " + queueName); - } -} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/error/RetryStrategy.java b/dotCMS/src/main/java/com/dotcms/jobs/business/error/RetryStrategy.java index 3c5996464718..23f54600de02 100644 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/error/RetryStrategy.java +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/error/RetryStrategy.java @@ -14,10 +14,10 @@ public interface RetryStrategy { * caused the failure. * * @param job The job that failed and is being considered for retry. - * @param exception The exception that caused the job to fail. + * @param exceptionClass The class of the exception that caused the failure. * @return true if the job should be retried, false otherwise. */ - boolean shouldRetry(Job job, Throwable exception); + boolean shouldRetry(Job job, Class exceptionClass); /** * Calculates the delay before the next retry attempt for a given job. @@ -37,17 +37,17 @@ public interface RetryStrategy { /** * Determines whether a given exception is retryable according to this strategy. * - * @param exception The exception to check. + * @param exceptionClass The class of the exception to check. * @return true if the exception is retryable, false otherwise. */ - boolean isRetryableException(Throwable exception); + boolean isRetryableException(Class exceptionClass); /** * Adds an exception class to the set of retryable exceptions. * * @param exceptionClass The exception class to be considered retryable. */ - void addRetryableException(final Class exceptionClass); + void addRetryableException(Class exceptionClass); /** * Returns an unmodifiable set of the currently registered retryable exceptions. diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/error/RetryStrategyProducer.java b/dotCMS/src/main/java/com/dotcms/jobs/business/error/RetryStrategyProducer.java index 1ad0da1f515c..ca3b48b8fb7e 100644 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/error/RetryStrategyProducer.java +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/error/RetryStrategyProducer.java @@ -28,7 +28,7 @@ public class RetryStrategyProducer { // The maximum number of retry attempts allowed static final int DEFAULT_RETRY_STRATEGY_MAX_RETRIES = Config.getIntProperty( - "DEFAULT_RETRY_STRATEGY_MAX_RETRIES", 5 + "DEFAULT_RETRY_STRATEGY_MAX_RETRIES", 3 ); /** diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/job/AbstractJob.java b/dotCMS/src/main/java/com/dotcms/jobs/business/job/AbstractJob.java index 149942d5a1e1..069f3fe09790 100644 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/job/AbstractJob.java +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/job/AbstractJob.java @@ -1,5 +1,6 @@ package com.dotcms.jobs.business.job; +import com.dotcms.jobs.business.processor.ProgressTracker; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.time.LocalDateTime; @@ -25,8 +26,12 @@ public interface AbstractJob { JobState state(); + Optional executionNode(); + Optional createdAt(); + Optional startedAt(); + Optional updatedAt(); Optional completedAt(); @@ -35,20 +40,13 @@ public interface AbstractJob { Map parameters(); - Optional lastException(); - - Optional errorDetail(); + Optional progressTracker(); @Default default int retryCount() { return 0; } - @Default - default long lastRetryTimestamp() { - return 0; - } - @Default default float progress() { return 0.0f; @@ -62,22 +60,35 @@ default float progress() { default Job incrementRetry() { return Job.builder().from(this) .retryCount(retryCount() + 1) - .lastRetryTimestamp(System.currentTimeMillis()) .build(); } /** - * Creates a new Job marked as failed with the given error detail. + * Creates a new Job marked as failed with the result details. * - * @param errorDetail The error detail to set. + * @param result The result details of the failed job. * @return A new Job instance marked as failed. */ - default Job markAsFailed(com.dotcms.jobs.business.error.ErrorDetail errorDetail) { + default Job markAsFailed(final JobResult result) { return Job.builder().from(this) .state(JobState.FAILED) - .result(JobResult.ERROR) - .errorDetail(errorDetail) - .lastException(errorDetail.exception()) + .result(result) + .completedAt(Optional.of(LocalDateTime.now())) + .updatedAt(LocalDateTime.now()) + .build(); + } + + /** + * Creates a new Job marked as running. + * + * @return A new Job instance marked as running. + */ + default Job markAsRunning() { + return Job.builder().from(this) + .state(JobState.RUNNING) + .result(Optional.empty()) + .startedAt(this.startedAt().orElse(LocalDateTime.now())) + .updatedAt(LocalDateTime.now()) .build(); } @@ -87,7 +98,7 @@ default Job markAsFailed(com.dotcms.jobs.business.error.ErrorDetail errorDetail) * @param newState The new state to set. * @return A new Job instance with the updated state. */ - default Job withState(JobState newState) { + default Job withState(final JobState newState) { return Job.builder().from(this) .state(newState) .updatedAt(LocalDateTime.now()) @@ -97,14 +108,34 @@ default Job withState(JobState newState) { /** * Creates a new Job marked as completed. * + * @param result The result details of the completed job. + * * @return A new Job instance marked as completed. */ - default Job markAsCompleted() { + default Job markAsCompleted(final JobResult result) { + return Job.builder().from(this) .state(JobState.COMPLETED) - .result(JobResult.SUCCESS) .completedAt(Optional.of(LocalDateTime.now())) .updatedAt(LocalDateTime.now()) + .result(result != null ? Optional.of(result) : Optional.empty()) + .build(); + } + + /** + * Creates a new Job marked as canceled. + * + * @param result The result details of the canceled job. + * + * @return A new Job instance marked as canceled. + */ + default Job markAsCanceled(final JobResult result) { + + return Job.builder().from(this) + .state(JobState.CANCELED) + .completedAt(Optional.of(LocalDateTime.now())) + .updatedAt(LocalDateTime.now()) + .result(result != null ? Optional.of(result) : Optional.empty()) .build(); } diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/job/AbstractJobPaginatedResult.java b/dotCMS/src/main/java/com/dotcms/jobs/business/job/AbstractJobPaginatedResult.java new file mode 100644 index 000000000000..6ae7d66a9ca0 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/job/AbstractJobPaginatedResult.java @@ -0,0 +1,22 @@ +package com.dotcms.jobs.business.job; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.List; +import org.immutables.value.Value; + +@Value.Style(typeImmutable = "*", typeAbstract = "Abstract*") +@Value.Immutable +@JsonSerialize(as = JobPaginatedResult.class) +@JsonDeserialize(as = JobPaginatedResult.class) +public interface AbstractJobPaginatedResult { + + List jobs(); + + long total(); + + int page(); + + int pageSize(); + +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/job/AbstractJobResult.java b/dotCMS/src/main/java/com/dotcms/jobs/business/job/AbstractJobResult.java new file mode 100644 index 000000000000..ecf1f2fa8d35 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/job/AbstractJobResult.java @@ -0,0 +1,24 @@ +package com.dotcms.jobs.business.job; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.Map; +import java.util.Optional; +import org.immutables.value.Value; + +/** + * Abstract interface for an immutable JobResult class. This interface defines the structure for job + * result information in the job processing system. The concrete implementation will be generated as + * an immutable class named JobResult. + */ +@Value.Style(typeImmutable = "*", typeAbstract = "Abstract*") +@Value.Immutable +@JsonSerialize(as = JobResult.class) +@JsonDeserialize(as = JobResult.class) +public interface AbstractJobResult { + + Optional errorDetail(); + + Optional> metadata(); + +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/job/JobResult.java b/dotCMS/src/main/java/com/dotcms/jobs/business/job/JobResult.java deleted file mode 100644 index cd89c2d811fb..000000000000 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/job/JobResult.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.dotcms.jobs.business.job; - -/** - * Represents the final result of a job execution. - */ -public enum JobResult { - - /** - * Indicates that the job completed successfully. - */ - SUCCESS, - - /** - * Indicates that the job encountered an error during execution. - */ - ERROR -} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/job/JobState.java b/dotCMS/src/main/java/com/dotcms/jobs/business/job/JobState.java index aa18e9fb9f1e..0bbdb4c50a0f 100644 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/job/JobState.java +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/job/JobState.java @@ -15,6 +15,11 @@ public enum JobState { */ RUNNING, + /** + * The job is currently being canceled. + */ + CANCELLING, + /** * The job has finished executing successfully. */ @@ -26,7 +31,7 @@ public enum JobState { FAILED, /** - * The job was cancelled before it could complete. + * The job was canceled before it could complete. */ - CANCELLED + CANCELED } \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/processor/Cancellable.java b/dotCMS/src/main/java/com/dotcms/jobs/business/processor/Cancellable.java new file mode 100644 index 000000000000..4465e3d85d23 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/processor/Cancellable.java @@ -0,0 +1,39 @@ +package com.dotcms.jobs.business.processor; + +import com.dotcms.jobs.business.error.JobCancellationException; +import com.dotcms.jobs.business.job.Job; + +/** + * The Cancellable interface represents a contract for objects that can be canceled, + * typically long-running operations or jobs. + *

+ * Implementations of this interface should provide a mechanism to interrupt or + * stop their execution in a controlled manner when the cancel method is invoked. + *

+ * It's important to note that implementing this interface indicates that the object + * supports cancellation. There is no separate method to check if cancellation is possible; + * the presence of this interface implies that it is. + */ +public interface Cancellable { + + /** + * Attempts to cancel the execution of this object. + *

+ * The exact behavior of this method depends on the specific implementation, + * but it should generally attempt to stop the ongoing operation as quickly + * and safely as possible. This might involve interrupting threads, closing + * resources, or setting flags to stop loops. + *

+ * Implementations should ensure that this method can be called safely from + * another thread while the operation is in progress. + *

+ * After this method is called, the object should make its best effort to + * terminate, but there's no guarantee about when the termination will occur. + * + * @throws JobCancellationException if there is an error during the cancellation process. + * This could occur if the job is in a state where it cannot be canceled, + * or if there's an unexpected error while attempting to cancel. + */ + void cancel(Job job) throws JobCancellationException; + +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/processor/JobProcessor.java b/dotCMS/src/main/java/com/dotcms/jobs/business/processor/JobProcessor.java index 9cd2fbc4c32d..8760008b1f4c 100644 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/processor/JobProcessor.java +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/processor/JobProcessor.java @@ -1,8 +1,8 @@ package com.dotcms.jobs.business.processor; -import com.dotcms.jobs.business.error.JobCancellationException; import com.dotcms.jobs.business.error.JobProcessingException; import com.dotcms.jobs.business.job.Job; +import java.util.Map; /** * Interface for processing jobs. Implementations of this interface should define how to process, @@ -19,30 +19,13 @@ public interface JobProcessor { void process(Job job) throws JobProcessingException; /** - * Determines if the given job can be cancelled. + * Returns metadata about the job execution. This metadata can be used to provide additional + * information about the job's execution, such as statistics or other details useful for the + * caller. * - * @param job The job to check for cancellation capability. - * @return true if the job can be cancelled, false otherwise. + * @param job The job for which to provide metadata. + * @return A map containing metadata about the job execution. */ - boolean canCancel(Job job); - - /** - * Cancels the given job. - * - * @param job The job to cancel. - * @throws JobCancellationException if an error occurs during cancellation. - */ - void cancel(Job job) throws JobCancellationException; - - /** - * Provides a progress tracker for the given job. The default implementation returns a new - * instance of DefaultProgressTracker. - * - * @param job The job for which to provide a progress tracker. - * @return A ProgressTracker instance for the given job. - */ - default ProgressTracker progressTracker(Job job) { - return new DefaultProgressTracker(); - } + Map getResultMetadata(Job job); } \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/queue/DBJobTransformer.java b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/DBJobTransformer.java new file mode 100644 index 000000000000..af2d83a17bb5 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/DBJobTransformer.java @@ -0,0 +1,267 @@ +package com.dotcms.jobs.business.queue; + +import com.dotcms.jobs.business.error.ErrorDetail; +import com.dotcms.jobs.business.job.Job; +import com.dotcms.jobs.business.job.JobResult; +import com.dotcms.jobs.business.job.JobState; +import com.dotmarketing.exception.DotRuntimeException; +import com.dotmarketing.util.UtilMethods; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import org.json.JSONException; +import org.json.JSONObject; +import org.postgresql.util.PGobject; + +/** + * Utility class for transforming database result sets into Job objects. This class provides static + * methods to convert raw database data into structured Job objects and their associated + * components. + */ +public class DBJobTransformer { + + private static final String ATTRIBUTE_RESULT_ERROR_DETAIL = "errorDetail"; + private static final String ATTRIBUTE_RESULT_METADATA = "metadata"; + + private DBJobTransformer() { + // Prevent instantiation + } + + /** + * Transforms a database result row into a Job object. + * + * @param row A map representing a row from the database result set + * @return A fully constructed Job object, or null if the input is null + */ + public static Job toJob(final Map row) { + if (row == null) { + return null; + } + + return Job.builder() + .id(getString(row, "id")) + .queueName(getString(row, "queue_name")) + .state(Objects.requireNonNull(getJobState(row))) + .parameters(getParameters(row)) + .result(getJobResult(row)) + .progress(getFloat(row, "progress")) + .createdAt(Objects.requireNonNull(getDateTime(row, "created_at"))) + .updatedAt(Objects.requireNonNull(getDateTime(row, "updated_at"))) + .startedAt(getOptionalDateTime(row, "started_at")) + .completedAt(getOptionalDateTime(row, "completed_at")) + .executionNode(getOptionalString(row, "execution_node")) + .retryCount(getInt(row, "retry_count")) + .build(); + } + + /** + * Retrieves a string value from the row map. + * + * @param row The row map + * @param column The column name + * @return The string value, or null if not present + */ + private static String getString(final Map row, final String column) { + return (String) row.get(column); + } + + /** + * Retrieves a JSON string value from the row map. + * + * @param row The row map + * @param column The column name + * @return The JSON string value, or null if not present + */ + private static String getJSONAsString( + final Map row, final String column) { + final var json = (PGobject) row.get(column); + return json != null ? json.getValue() : null; + } + + /** + * Retrieves an optional string value from the row map. + * + * @param row The row map + * @param column The column name + * @return An Optional containing the string value, or empty if not present + */ + private static Optional getOptionalString(final Map row, + final String column) { + return Optional.ofNullable(getString(row, column)); + } + + /** + * Retrieves the JobState from the row map. + * + * @param row The row map + * @return The JobState, or null if not present or invalid + */ + private static JobState getJobState(final Map row) { + String stateStr = getString(row, "state"); + return UtilMethods.isSet(stateStr) ? JobState.valueOf(stateStr) : null; + } + + /** + * Retrieves the job parameters from the row map. + * + * @param row The row map + * @return A Map of job parameters, or an empty map if not present or invalid + */ + private static Map getParameters(final Map row) { + + String paramsJson = getJSONAsString(row, "parameters"); + if (!UtilMethods.isSet(paramsJson)) { + return new HashMap<>(); + } + + try { + JSONObject jsonObject = new JSONObject(paramsJson); + return jsonObject.toMap(); + } catch (JSONException e) { + throw new DotRuntimeException("Error parsing job parameters", e); + } + } + + /** + * Retrieves the JobResult from the row map. + * + * @param row The row map + * @return An Optional containing the JobResult, or empty if not present or invalid + */ + private static Optional getJobResult(final Map row) { + + String resultJson = getJSONAsString(row, "result"); + if (!UtilMethods.isSet(resultJson)) { + return Optional.empty(); + } + + try { + JSONObject resultJsonObject = new JSONObject(resultJson); + return Optional.of(JobResult.builder() + .errorDetail(getErrorDetail(resultJsonObject)) + .metadata(getMetadata(resultJsonObject)) + .build()); + } catch (JSONException e) { + throw new DotRuntimeException("Error parsing job result", e); + } + } + + /** + * Retrieves the ErrorDetail from a JSON object. + * + * @param resultJsonObject The JSON object containing error details + * @return An Optional containing the ErrorDetail, or empty if not present or invalid + */ + private static Optional getErrorDetail(final JSONObject resultJsonObject) { + + if (!resultJsonObject.has(ATTRIBUTE_RESULT_ERROR_DETAIL)) { + return Optional.empty(); + } + + try { + if (resultJsonObject.isNull(ATTRIBUTE_RESULT_ERROR_DETAIL)) { + return Optional.empty(); + } + + JSONObject errorDetailJson = resultJsonObject.getJSONObject(ATTRIBUTE_RESULT_ERROR_DETAIL); + return Optional.of(ErrorDetail.builder() + .message(errorDetailJson.optString("message")) + .exceptionClass(errorDetailJson.optString("exceptionClass")) + .timestamp(Objects.requireNonNull(getDateTime(errorDetailJson.opt("timestamp")))) + .processingStage(errorDetailJson.optString("processingStage")) + .stackTrace(errorDetailJson.optString("stackTrace")) + .build()); + } catch (JSONException e) { + throw new DotRuntimeException("Error parsing error detail", e); + } + } + + /** + * Retrieves the metadata from a JSON object. + * + * @param resultJsonObject The JSON object containing metadata + * @return An Optional containing the metadata as a Map, or empty if not present or invalid + */ + private static Optional> getMetadata(final JSONObject resultJsonObject) { + + if (!resultJsonObject.has(ATTRIBUTE_RESULT_METADATA)) { + return Optional.empty(); + } + + try { + if (resultJsonObject.isNull(ATTRIBUTE_RESULT_METADATA)) { + return Optional.empty(); + } + return Optional.of(resultJsonObject.getJSONObject(ATTRIBUTE_RESULT_METADATA).toMap()); + } catch (JSONException e) { + throw new DotRuntimeException("Error parsing metadata", e); + } + } + + /** + * Retrieves a float value from the row map. + * + * @param row The row map + * @param column The column name + * @return The float value, or 0 if not present or invalid + */ + private static float getFloat(final Map row, final String column) { + Object value = row.get(column); + return value instanceof Number ? ((Number) value).floatValue() : 0f; + } + + /** + * Retrieves an integer value from the row map. + * + * @param row The row map + * @param column The column name + * @return The integer value, or 0 if not present or invalid + */ + private static int getInt(final Map row, final String column) { + Object value = row.get(column); + return value instanceof Number ? ((Number) value).intValue() : 0; + } + + /** + * Retrieves a LocalDateTime value from the row map. + * + * @param row The row map + * @param column The column name + * @return The LocalDateTime value, or null if not present or invalid + */ + private static LocalDateTime getDateTime(final Map row, final String column) { + Object value = row.get(column); + return getDateTime(value); + } + + /** + * Converts an Object to a LocalDateTime. + * + * @param value The object to convert + * @return The LocalDateTime value, or null if the input is not a Timestamp + */ + private static LocalDateTime getDateTime(final Object value) { + if (value instanceof Timestamp) { + return ((Timestamp) value).toLocalDateTime(); + } else if (value instanceof String) { + return LocalDateTime.parse((String) value); + } + return null; + } + + /** + * Retrieves an optional LocalDateTime value from the row map. + * + * @param row The row map + * @param column The column name + * @return An Optional containing the LocalDateTime value, or empty if not present or invalid + */ + private static Optional getOptionalDateTime(final Map row, + final String column) { + return Optional.ofNullable(getDateTime(row, column)); + } + +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/queue/JobQueue.java b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/JobQueue.java index b3a165ef1b8e..45cdebfabcf9 100644 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/queue/JobQueue.java +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/JobQueue.java @@ -1,9 +1,16 @@ package com.dotcms.jobs.business.queue; import com.dotcms.jobs.business.job.Job; +import com.dotcms.jobs.business.job.JobPaginatedResult; +import com.dotcms.jobs.business.job.JobState; +import com.dotcms.jobs.business.queue.error.JobLockingException; +import com.dotcms.jobs.business.queue.error.JobNotFoundException; +import com.dotcms.jobs.business.queue.error.JobQueueDataException; +import com.dotcms.jobs.business.queue.error.JobQueueException; import java.time.LocalDateTime; import java.util.List; import java.util.Map; +import java.util.Set; /** * Defines the contract for a job queue system. This interface provides methods for adding, @@ -12,21 +19,26 @@ public interface JobQueue { /** - * Adds a new job to the specified queue. + * Creates a new job in the specified queue. * - * @param queueName The name of the queue to add the job to. - * @param parameters The parameters for the job. + * @param queueName The name of the queue to add the job to. + * @param parameters The parameters for the job. * @return The ID of the newly created job. + * @throws JobQueueException if there's an error creating the job + * @throws JobQueueDataException if there's a data storage error while creating the job */ - String addJob(String queueName, Map parameters); + String createJob(String queueName, Map parameters) + throws JobQueueException; /** * Retrieves a job by its ID. * * @param jobId The ID of the job to retrieve. - * @return The job with the specified ID, or null if not found. + * @return The job with the specified ID. + * @throws JobNotFoundException if the job with the given ID is not found + * @throws JobQueueDataException if there's a data storage error while fetching the job */ - Job getJob(String jobId); + Job getJob(String jobId) throws JobNotFoundException, JobQueueDataException; /** * Retrieves a list of active jobs for a specific queue. @@ -34,9 +46,11 @@ public interface JobQueue { * @param queueName The name of the queue. * @param page The page number (for pagination). * @param pageSize The number of items per page. - * @return A list of active jobs. + * @return A result object containing the list of active jobs and pagination information. + * @throws JobQueueDataException if there's a data storage error while fetching the jobs */ - List getActiveJobs(String queueName, int page, int pageSize); + JobPaginatedResult getActiveJobs(String queueName, int page, int pageSize) + throws JobQueueDataException; /** * Retrieves a list of completed jobs for a specific queue within a date range. @@ -46,57 +60,77 @@ public interface JobQueue { * @param endDate The end date of the range. * @param page The page number (for pagination). * @param pageSize The number of items per page. - * @return A list of completed jobs. + * @return A result object containing the list of active jobs and pagination information. + * @throws JobQueueDataException if there's a data storage error while fetching the jobs */ - List getCompletedJobs(String queueName, LocalDateTime startDate, LocalDateTime endDate, - int page, int pageSize); + JobPaginatedResult getCompletedJobs(String queueName, LocalDateTime startDate, + LocalDateTime endDate, int page, int pageSize) throws JobQueueDataException; /** * Retrieves a list of all jobs. * * @param page The page number (for pagination). * @param pageSize The number of items per page. - * @return A list of all jobs. + * @return A result object containing the list of jobs and pagination information. + * @throws JobQueueDataException if there's a data storage error while fetching the jobs */ - List getJobs(int page, int pageSize); + JobPaginatedResult getJobs(int page, int pageSize) throws JobQueueDataException; /** * Retrieves a list of failed jobs. * * @param page The page number (for pagination). * @param pageSize The number of items per page. - * @return A list of failed jobs. + * @return A result object containing the list of failed jobs and pagination information. + * @throws JobQueueDataException if there's a data storage error while fetching the jobs */ - List getFailedJobs(int page, int pageSize); + JobPaginatedResult getFailedJobs(int page, int pageSize) throws JobQueueDataException; /** * Updates the status of a job. * * @param job The job with an updated status. + * @throws JobQueueDataException if there's a data storage error while updating the job status */ - void updateJobStatus(Job job); + void updateJobStatus(Job job) throws JobQueueDataException; + + /** + * Retrieves updates for specific jobs since a given time. + * + * @param jobIds The IDs of the jobs to check for updates + * @param since The time from which to fetch updates + * @return A list of updated Job objects + * @throws JobQueueDataException if there's a data storage error while fetching job updates + */ + List getUpdatedJobsSince(Set jobIds, LocalDateTime since) + throws JobQueueDataException; /** * Puts a job back in the queue for retry. * * @param job The job to retry. + * @throws JobQueueDataException if there's a data storage error while re-queueing the job */ - void putJobBackInQueue(Job job); + void putJobBackInQueue(Job job) throws JobQueueDataException; /** * Retrieves the next job in the queue. * * @return The next job in the queue, or null if the queue is empty. + * @throws JobQueueDataException if there's a data storage error while fetching the next job + * @throws JobLockingException if there's an error acquiring a lock on the next job */ - Job nextJob(); + Job nextJob() throws JobQueueDataException, JobLockingException; /** * Updates the progress of a job. * * @param jobId The ID of the job to update. * @param progress The new progress value (between 0.0 and 1.0). + * @throws JobQueueDataException if there's a data storage error while updating the job + * progress */ - void updateJobProgress(String jobId, float progress); + void updateJobProgress(String jobId, float progress) throws JobQueueDataException; /** * Removes a job from the queue. This method should be used for jobs that have permanently @@ -104,8 +138,18 @@ List getCompletedJobs(String queueName, LocalDateTime startDate, LocalDateT * removed from the queue and any associated resources are cleaned up. * * @param jobId The ID of the job to remove. - * @throws IllegalArgumentException if the job with the given ID does not exist. + * @throws JobQueueDataException if there's a data storage error while removing the job + */ + void removeJobFromQueue(String jobId) throws JobQueueDataException; + + /** + * Checks if a job has ever been in a specific state. + * + * @param jobId The ID of the job to check. + * @param state The state to check for. + * @return true if the job has been in the specified state, false otherwise. + * @throws JobQueueDataException if there's an error accessing the job data. */ - void removeJob(String jobId); + boolean hasJobBeenInState(String jobId, JobState state) throws JobQueueDataException; } \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/queue/JobQueueProducer.java b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/JobQueueProducer.java index 210653cadc56..1596e9fb17d6 100644 --- a/dotCMS/src/main/java/com/dotcms/jobs/business/queue/JobQueueProducer.java +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/JobQueueProducer.java @@ -1,5 +1,6 @@ package com.dotcms.jobs.business.queue; +import com.dotmarketing.util.Config; import javax.enterprise.context.ApplicationScoped; import javax.enterprise.inject.Produces; @@ -10,6 +11,11 @@ @ApplicationScoped public class JobQueueProducer { + // The type of job queue implementation to use + private static final String JOB_QUEUE_IMPLEMENTATION_TYPE = Config.getStringProperty( + "JOB_QUEUE_IMPLEMENTATION_TYPE", "postgres" + ); + /** * Produces a JobQueue instance. This method is called by the CDI container to create a JobQueue * instance when it is needed for dependency injection. @@ -20,17 +26,13 @@ public class JobQueueProducer { @ApplicationScoped public JobQueue produceJobQueue() { - // Potential future implementation: - // String queueType = System.getProperty("job.queue.type", "postgres"); - // if ("postgres".equals(queueType)) { - // return new PostgresJobQueue(); - // } else if ("redis".equals(queueType)) { - // return new RedisJobQueue(); - // } - // throw new IllegalStateException("Unknown job queue type: " + queueType); + if (JOB_QUEUE_IMPLEMENTATION_TYPE.equals("postgres")) { + return new PostgresJobQueue(); + } - //return new PostgresJobQueue(); - return null; + throw new IllegalStateException( + "Unknown job queue implementation type: " + JOB_QUEUE_IMPLEMENTATION_TYPE + ); } } \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/queue/PostgresJobQueue.java b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/PostgresJobQueue.java new file mode 100644 index 000000000000..37bb0fa9ac1e --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/PostgresJobQueue.java @@ -0,0 +1,545 @@ +package com.dotcms.jobs.business.queue; + +import com.dotcms.jobs.business.job.Job; +import com.dotcms.jobs.business.job.JobPaginatedResult; +import com.dotcms.jobs.business.job.JobState; +import com.dotcms.jobs.business.queue.error.JobLockingException; +import com.dotcms.jobs.business.queue.error.JobNotFoundException; +import com.dotcms.jobs.business.queue.error.JobQueueDataException; +import com.dotcms.jobs.business.queue.error.JobQueueException; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.common.db.DotConnect; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.util.Logger; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.guava.GuavaModule; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.github.jonpeterson.jackson.module.versioning.VersioningModule; +import io.vavr.Lazy; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +/** + * PostgreSQL implementation of the JobQueue interface. This class provides concrete implementations + * for managing jobs using a PostgreSQL database. + * + *

The PostgresJobQueue handles all database operations related to job management, including: + *

    + *
  • Creating new jobs and adding them to the queue
  • + *
  • Retrieving jobs by various criteria (ID, status, date range)
  • + *
  • Updating job status and progress
  • + *
  • Managing job lifecycle (queueing, processing, completion, failure)
  • + *
  • Implementing job locking mechanism for concurrent processing
  • + *
+ * + *

This implementation uses SQL queries optimized for PostgreSQL, including the use of + * `SELECT FOR UPDATE SKIP LOCKED` for efficient job queue management in concurrent environments. + * + *

Note: This class assumes the existence of appropriate database tables (job_queue, job, + * job_history) as defined in the database schema. Ensure that these tables are properly + * set up before using this class. + * + * @see JobQueue + * @see Job + * @see JobState + */ +public class PostgresJobQueue implements JobQueue { + + private static final String CREATE_JOB_QUEUE_QUERY = "INSERT INTO job_queue " + + "(id, queue_name, state, created_at) VALUES (?, ?, ?, ?)"; + + private static final String CREATE_JOB_QUERY = "INSERT INTO job " + + "(id, queue_name, state, parameters, created_at, execution_node, updated_at) " + + "VALUES (?, ?, ?, ?::jsonb, ?, ?, ?)"; + + private static final String CREATE_JOB_HISTORY_QUERY = + "INSERT INTO job_history " + + "(id, job_id, state, execution_node, created_at) " + + "VALUES (?, ?, ?, ?, ?)"; + + private static final String SELECT_JOB_BY_ID_QUERY = "SELECT * FROM job WHERE id = ?"; + + private static final String UPDATE_AND_GET_NEXT_JOB_WITH_LOCK_QUERY = + "UPDATE job_queue SET state = ? " + + "WHERE id = (SELECT id FROM job_queue WHERE state = ? " + + "ORDER BY priority DESC, created_at ASC LIMIT 1 FOR UPDATE SKIP LOCKED) " + + "RETURNING *"; + + private static final String GET_ACTIVE_JOBS_QUERY = + "WITH total AS (SELECT COUNT(*) AS total_count " + + " FROM job WHERE queue_name = ? AND state IN (?, ?) " + + "), " + + "paginated_data AS (SELECT * " + + " FROM job WHERE queue_name = ? AND state IN (?, ?) " + + " ORDER BY created_at LIMIT ? OFFSET ? " + + ") " + + "SELECT p.*, t.total_count FROM total t LEFT JOIN paginated_data p ON true"; + + private static final String GET_COMPLETED_JOBS_QUERY = + "WITH total AS (SELECT COUNT(*) AS total_count " + + " FROM job WHERE queue_name = ? AND state = ? AND completed_at BETWEEN ? AND ? " + + "), " + + "paginated_data AS (SELECT * FROM job " + + " WHERE queue_name = ? AND state = ? AND completed_at BETWEEN ? AND ? " + + " ORDER BY completed_at DESC LIMIT ? OFFSET ? " + + ") " + + "SELECT p.*, t.total_count FROM total t LEFT JOIN paginated_data p ON true"; + + private static final String GET_FAILED_JOBS_QUERY = + "WITH total AS (" + + " SELECT COUNT(*) AS total_count FROM job " + + " WHERE state = ? " + + "), " + + "paginated_data AS (" + + " SELECT * " + + " FROM job WHERE state = ? " + + " ORDER BY updated_at DESC " + + " LIMIT ? OFFSET ? " + + ") " + + "SELECT p.*, t.total_count " + + "FROM total t " + + "LEFT JOIN paginated_data p ON true"; + + private static final String GET_JOBS_QUERY = + "WITH total AS (" + + " SELECT COUNT(*) AS total_count " + + " FROM job " + + "), " + + "paginated_data AS (" + + " SELECT * " + + " FROM job " + + " ORDER BY created_at DESC " + + " LIMIT ? OFFSET ? " + + ") " + + "SELECT p.*, t.total_count " + + "FROM total t " + + "LEFT JOIN paginated_data p ON true"; + + private static final String GET_UPDATED_JOBS_SINCE_QUERY = "SELECT * FROM job " + + "WHERE id = ANY(?) AND updated_at > ?"; + + private static final String UPDATE_JOBS_QUERY = "UPDATE job SET state = ?, progress = ?, " + + "updated_at = ?, started_at = ?, completed_at = ?, execution_node = ?, retry_count = ?, " + + "result = ?::jsonb WHERE id = ?"; + + private static final String INSERT_INTO_JOB_HISTORY_QUERY = + "INSERT INTO job_history " + + "(id, job_id, state, execution_node, created_at, result) " + + "VALUES (?, ?, ?, ?, ?, ?::jsonb)"; + + private static final String DELETE_JOB_FROM_QUEUE_QUERY = "DELETE FROM job_queue WHERE id = ?"; + + private static final String PUT_JOB_BACK_TO_QUEUE_QUERY = "INSERT INTO job_queue " + + "(id, queue_name, state, priority, created_at) " + + "VALUES (?, ?, ?, ?, ?) ON CONFLICT (id) DO UPDATE SET state = ?, priority = ?"; + + private static final String UPDATE_JOB_PROGRESS_QUERY = + "UPDATE job SET progress = ?, updated_at = ?" + + " WHERE id = ?"; + + private static final String HAS_JOB_BEEN_IN_STATE_QUERY = "SELECT " + + "EXISTS (SELECT 1 FROM job_history WHERE job_id = ? AND state = ?)"; + + private static final String COLUMN_TOTAL_COUNT = "total_count"; + + /** + * Jackson mapper configuration and lazy initialized instance. + */ + private final Lazy objectMapper = Lazy.of(() -> { + ObjectMapper mapper = new ObjectMapper(); + mapper.enable(SerializationFeature.INDENT_OUTPUT); + mapper.registerModule(new Jdk8Module()); + mapper.registerModule(new GuavaModule()); + mapper.registerModule(new JavaTimeModule()); + mapper.registerModule(new VersioningModule()); + mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + return mapper; + }); + + @Override + public String createJob(final String queueName, final Map parameters) + throws JobQueueException { + + final String serverId = APILocator.getServerAPI().readServerId(); + + try { + + final String jobId = UUID.randomUUID().toString(); + + final var parametersJson = objectMapper.get().writeValueAsString(parameters); + final var now = Timestamp.valueOf(LocalDateTime.now()); + final var jobState = JobState.PENDING.name(); + + // Insert into job table + new DotConnect().setSQL(CREATE_JOB_QUERY) + .addParam(jobId) + .addParam(queueName) + .addParam(jobState) + .addParam(parametersJson) + .addParam(now) + .addParam(serverId) + .addParam(now) + .loadResult(); + + // Creating the jobqueue entry + new DotConnect().setSQL(CREATE_JOB_QUEUE_QUERY) + .addParam(jobId) + .addParam(queueName) + .addParam(jobState) + .addParam(now) + .loadResult(); + + // Creating job_history entry + new DotConnect().setSQL(CREATE_JOB_HISTORY_QUERY) + .addParam(UUID.randomUUID().toString()) + .addParam(jobId) + .addParam(jobState) + .addParam(serverId) + .addParam(now) + .loadResult(); + + return jobId; + } catch (JsonProcessingException e) { + Logger.error(this, "Failed to serialize job parameters", e); + throw new JobQueueException("Failed to serialize job parameters", e); + } catch (DotDataException e) { + Logger.error(this, "Database error while creating job", e); + throw new JobQueueDataException("Database error while creating job", e); + } + } + + @Override + public Job getJob(final String jobId) throws JobNotFoundException, JobQueueDataException { + + try { + DotConnect dc = new DotConnect(); + dc.setSQL(SELECT_JOB_BY_ID_QUERY); + dc.addParam(jobId); + + List> results = dc.loadObjectResults(); + if (!results.isEmpty()) { + return DBJobTransformer.toJob(results.get(0)); + } + + Logger.warn(this, "Job with id: " + jobId + " not found"); + throw new JobNotFoundException(jobId); + } catch (DotDataException e) { + Logger.error(this, "Database error while fetching job", e); + throw new JobQueueDataException("Database error while fetching job", e); + } + } + + @Override + public JobPaginatedResult getActiveJobs(final String queueName, final int page, + final int pageSize) throws JobQueueDataException { + + try { + + DotConnect dc = new DotConnect(); + dc.setSQL(GET_ACTIVE_JOBS_QUERY); + dc.addParam(queueName); + dc.addParam(JobState.PENDING.name()); + dc.addParam(JobState.RUNNING.name()); + dc.addParam(queueName); // Repeated for paginated_data CTE + dc.addParam(JobState.PENDING.name()); + dc.addParam(JobState.RUNNING.name()); + dc.addParam(pageSize); + dc.addParam((page - 1) * pageSize); + + return jobPaginatedResult(page, pageSize, dc); + } catch (DotDataException e) { + Logger.error(this, "Database error while fetching active jobs", e); + throw new JobQueueDataException("Database error while fetching active jobs", e); + } + } + + @Override + public JobPaginatedResult getCompletedJobs(final String queueName, + final LocalDateTime startDate, + final LocalDateTime endDate, final int page, final int pageSize) + throws JobQueueDataException { + + try { + DotConnect dc = new DotConnect(); + dc.setSQL(GET_COMPLETED_JOBS_QUERY); + dc.addParam(queueName); + dc.addParam(JobState.COMPLETED.name()); + dc.addParam(Timestamp.valueOf(startDate)); + dc.addParam(Timestamp.valueOf(endDate)); + dc.addParam(queueName); // Repeated for paginated_data CTE + dc.addParam(JobState.COMPLETED.name()); + dc.addParam(Timestamp.valueOf(startDate)); + dc.addParam(Timestamp.valueOf(endDate)); + dc.addParam(pageSize); + dc.addParam((page - 1) * pageSize); + + return jobPaginatedResult(page, pageSize, dc); + } catch (DotDataException e) { + Logger.error(this, "Database error while fetching completed jobs", e); + throw new JobQueueDataException("Database error while fetching completed jobs", e); + } + } + + @Override + public JobPaginatedResult getJobs(final int page, final int pageSize) + throws JobQueueDataException { + + try { + DotConnect dc = new DotConnect(); + dc.setSQL(GET_JOBS_QUERY); + dc.addParam(pageSize); + dc.addParam((page - 1) * pageSize); + + return jobPaginatedResult(page, pageSize, dc); + } catch (DotDataException e) { + Logger.error(this, "Database error while fetching jobs", e); + throw new JobQueueDataException("Database error while fetching jobs", e); + } + } + + @Override + public JobPaginatedResult getFailedJobs(final int page, final int pageSize) + throws JobQueueDataException { + + try { + DotConnect dc = new DotConnect(); + dc.setSQL(GET_FAILED_JOBS_QUERY); + dc.addParam(JobState.FAILED.name()); + dc.addParam(JobState.FAILED.name()); // Repeated for paginated_data CTE + dc.addParam(pageSize); + dc.addParam((page - 1) * pageSize); + + return jobPaginatedResult(page, pageSize, dc); + } catch (DotDataException e) { + Logger.error(this, "Database error while fetching failed jobs", e); + throw new JobQueueDataException("Database error while fetching failed jobs", e); + } + } + + @Override + public void updateJobStatus(final Job job) throws JobQueueDataException { + + final String serverId = APILocator.getServerAPI().readServerId(); + + try { + + DotConnect dc = new DotConnect(); + dc.setSQL(UPDATE_JOBS_QUERY); + dc.addParam(job.state().name()); + dc.addParam(job.progress()); + dc.addParam(Timestamp.valueOf(LocalDateTime.now())); + dc.addParam(job.startedAt().map(Timestamp::valueOf).orElse(null)); + dc.addParam(job.completedAt().map(Timestamp::valueOf).orElse(null)); + dc.addParam(serverId); + dc.addParam(job.retryCount()); + dc.addParam(job.result().map(r -> { + try { + return objectMapper.get().writeValueAsString(r); + } catch (Exception e) { + Logger.error(this, "Failed to serialize job result", e); + return null; + } + }).orElse(null)); + dc.addParam(job.id()); + dc.loadResult(); + + // Update job_history + DotConnect historyDc = new DotConnect(); + historyDc.setSQL(INSERT_INTO_JOB_HISTORY_QUERY); + historyDc.addParam(UUID.randomUUID().toString()); + historyDc.addParam(job.id()); + historyDc.addParam(job.state().name()); + historyDc.addParam(serverId); + historyDc.addParam(Timestamp.valueOf(LocalDateTime.now())); + historyDc.addParam(job.result().map(r -> { + try { + return objectMapper.get().writeValueAsString(r); + } catch (Exception e) { + Logger.error(this, "Failed to serialize job result for history", e); + return null; + } + }).orElse(null)); + historyDc.loadResult(); + + // Remove from job_queue if completed, failed, or canceled + if (job.state() == JobState.COMPLETED + || job.state() == JobState.FAILED + || job.state() == JobState.CANCELED) { + removeJobFromQueue(job.id()); + } + } catch (DotDataException e) { + Logger.error(this, "Database error while updating job status", e); + throw new JobQueueDataException("Database error while updating job status", e); + } + } + + @Override + public List getUpdatedJobsSince(final Set jobIds, final LocalDateTime since) + throws JobQueueDataException { + + try { + if (jobIds.isEmpty()) { + return Collections.emptyList(); + } + + DotConnect dc = new DotConnect(); + dc.setSQL(GET_UPDATED_JOBS_SINCE_QUERY); + dc.addParam(jobIds.toArray(new String[0])); + dc.addParam(Timestamp.valueOf(since)); + + List> results = dc.loadObjectResults(); + return results.stream().map(DBJobTransformer::toJob).collect(Collectors.toList()); + } catch (DotDataException e) { + Logger.error(this, "Database error while fetching updated jobs", e); + throw new JobQueueDataException("Database error while fetching updated jobs", e); + } + } + + @Override + public void putJobBackInQueue(final Job job) throws JobQueueDataException { + + try { + DotConnect dc = new DotConnect(); + dc.setSQL(PUT_JOB_BACK_TO_QUEUE_QUERY); + dc.addParam(job.id()); + dc.addParam(job.queueName()); + dc.addParam(JobState.PENDING.name()); + dc.addParam(0); // Default priority + dc.addParam(Timestamp.valueOf(LocalDateTime.now())); + dc.addParam(JobState.PENDING.name()); + dc.addParam(0); // Default priority + dc.loadResult(); + } catch (DotDataException e) { + Logger.error(this, "Database error while putting job back in queue", e); + throw new JobQueueDataException( + "Database error while putting job back in queue", e + ); + } + } + + @Override + public Job nextJob() throws JobQueueDataException, JobLockingException { + + try { + DotConnect dc = new DotConnect(); + dc.setSQL(UPDATE_AND_GET_NEXT_JOB_WITH_LOCK_QUERY); + dc.addParam(JobState.RUNNING.name()); + dc.addParam(JobState.PENDING.name()); + + List> results = dc.loadObjectResults(); + if (!results.isEmpty()) { + + // Fetch full job details from the job table + String jobId = (String) results.get(0).get("id"); + return getJob(jobId); + } + + return null; + } catch (DotDataException e) { + Logger.error(this, "Database error while fetching next job", e); + throw new JobQueueDataException("Database error while fetching next job", e); + } catch (JobNotFoundException e) { + Logger.error(this, "Job not found while fetching next job", e); + throw new JobQueueDataException("Job not found while fetching next job", e); + } catch (Exception e) { + Logger.error(this, "Error while locking next job", e); + throw new JobLockingException("Error while locking next job: " + e.getMessage()); + } + } + + @Override + public void updateJobProgress(final String jobId, final float progress) + throws JobQueueDataException { + + try { + DotConnect dc = new DotConnect(); + dc.setSQL(UPDATE_JOB_PROGRESS_QUERY); + dc.addParam(progress); + dc.addParam(Timestamp.valueOf(LocalDateTime.now())); + dc.addParam(jobId); + dc.loadResult(); + } catch (DotDataException e) { + Logger.error(this, "Database error while updating job progress", e); + throw new JobQueueDataException("Database error while updating job progress", e); + } + } + + @Override + public void removeJobFromQueue(final String jobId) throws JobQueueDataException { + + try { + DotConnect dc = new DotConnect(); + dc.setSQL(DELETE_JOB_FROM_QUEUE_QUERY); + dc.addParam(jobId); + dc.loadResult(); + } catch (DotDataException e) { + Logger.error(this, "Database error while removing job", e); + throw new JobQueueDataException("Database error while removing job", e); + } + } + + @Override + public boolean hasJobBeenInState(String jobId, JobState state) throws JobQueueDataException { + + try { + DotConnect dc = new DotConnect(); + dc.setSQL(HAS_JOB_BEEN_IN_STATE_QUERY); + dc.addParam(jobId); + dc.addParam(state.name()); + List> results = dc.loadObjectResults(); + + if (!results.isEmpty()) { + return (Boolean) results.get(0).get("exists"); + } + + return false; + } catch (Exception e) { + Logger.error(this, "Error checking job state history", e); + throw new JobQueueDataException("Error checking job state history", e); + } + } + + /** + * Helper method to create a JobPaginatedResult from a DotConnect query result. + * + * @param page The current page number + * @param pageSize The number of items per page + * @param dc The DotConnect instance with the query results + * @return A JobPaginatedResult instance + * @throws DotDataException If there is an error loading the query results + */ + private static JobPaginatedResult jobPaginatedResult( + int page, int pageSize, DotConnect dc) throws DotDataException { + + final var results = dc.loadObjectResults(); + + long totalCount = 0; + List jobs = new ArrayList<>(); + + if (!results.isEmpty()) { + totalCount = ((Number) results.get(0).get(COLUMN_TOTAL_COUNT)).longValue(); + jobs = results.stream() + .filter(row -> row.get("id") != null) // Filter out rows without job data + .map(DBJobTransformer::toJob) + .collect(Collectors.toList()); + } + + return JobPaginatedResult.builder() + .jobs(jobs) + .total(totalCount) + .page(page) + .pageSize(pageSize) + .build(); + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobLockingException.java b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobLockingException.java new file mode 100644 index 000000000000..dc6f8418182d --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobLockingException.java @@ -0,0 +1,18 @@ +package com.dotcms.jobs.business.queue.error; + +/** + * Exception thrown when there's an error in acquiring or managing locks for job processing. This + * could occur during attempts to atomically fetch and lock the next job for processing. + */ +public class JobLockingException extends JobQueueException { + + /** + * Constructs a new JobLockingException with the specified detail message. + * + * @param message the detail message + */ + public JobLockingException(String message) { + super(message); + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobNotFoundException.java b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobNotFoundException.java new file mode 100644 index 000000000000..977cf8198cf3 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobNotFoundException.java @@ -0,0 +1,19 @@ +package com.dotcms.jobs.business.queue.error; + +/** + * Exception thrown when a requested job cannot be found in the job queue. This typically occurs + * when trying to retrieve, update, or process a job that no longer exists. + */ +public class JobNotFoundException extends JobQueueException { + + /** + * Constructs a new JobNotFoundException with a message indicating the missing job's ID. + * + * @param jobId the ID of the job that was not found + */ + public JobNotFoundException(String jobId) { + super("Job with id: " + jobId + " not found"); + } + +} + diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobQueueDataException.java b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobQueueDataException.java new file mode 100644 index 000000000000..9075690ac129 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobQueueDataException.java @@ -0,0 +1,29 @@ +package com.dotcms.jobs.business.queue.error; + +/** + * Exception thrown when a data-related error occurs during job queue operations. This could include + * connection issues, query failures, or data integrity problems, regardless of the underlying + * storage mechanism (e.g., database, in-memory store, distributed cache). + */ +public class JobQueueDataException extends JobQueueException { + + /** + * Constructs a new JobQueueDataException with the specified detail message. + * + * @param message the detail message + */ + public JobQueueDataException(String message) { + super(message); + } + + /** + * Constructs a new JobQueueDataException with the specified detail message and cause. + * + * @param message the detail message + * @param cause the cause of this exception + */ + public JobQueueDataException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobQueueException.java b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobQueueException.java new file mode 100644 index 000000000000..4ddd1af4a243 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobQueueException.java @@ -0,0 +1,28 @@ +package com.dotcms.jobs.business.queue.error; + +/** + * Base exception class for all job queue related errors. This exception is the parent of all more + * specific job queue exceptions. + */ +public class JobQueueException extends Exception { + + /** + * Constructs a new JobQueueException with the specified detail message. + * + * @param message the detail message + */ + public JobQueueException(String message) { + super(message); + } + + /** + * Constructs a new JobQueueException with the specified detail message and cause. + * + * @param message the detail message + * @param cause the cause of this exception + */ + public JobQueueException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobQueueFullException.java b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobQueueFullException.java new file mode 100644 index 000000000000..7f685d023323 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/jobs/business/queue/error/JobQueueFullException.java @@ -0,0 +1,18 @@ +package com.dotcms.jobs.business.queue.error; + +/** + * Exception thrown when the job queue has reached its capacity and cannot accept new jobs. This may + * occur if there's a limit on the number of pending jobs or if system resources are exhausted. + */ +public class JobQueueFullException extends JobQueueException { + + /** + * Constructs a new JobQueueFullException with the specified detail message. + * + * @param message the detail message + */ + public JobQueueFullException(String message) { + super(message); + } + +} diff --git a/dotCMS/src/main/java/com/dotmarketing/business/APILocator.java b/dotCMS/src/main/java/com/dotmarketing/business/APILocator.java index f3f10b5051c3..07921109e3fb 100644 --- a/dotCMS/src/main/java/com/dotmarketing/business/APILocator.java +++ b/dotCMS/src/main/java/com/dotmarketing/business/APILocator.java @@ -62,6 +62,7 @@ import com.dotcms.experiments.business.ExperimentsAPIImpl; import com.dotcms.graphql.business.GraphqlAPI; import com.dotcms.graphql.business.GraphqlAPIImpl; +import com.dotcms.jobs.business.api.JobQueueManagerAPI; import com.dotcms.keyvalue.business.KeyValueAPI; import com.dotcms.keyvalue.business.KeyValueAPIImpl; import com.dotcms.languagevariable.business.LanguageVariableAPI; @@ -285,6 +286,15 @@ public static DotAIAPI getDotAIAPI() { return (DotAIAPI)getInstance(APIIndex.ARTIFICIAL_INTELLIGENCE_API); } + /** + * Creates a single instance of the {@link JobQueueManagerAPI} class. + * + * @return The {@link JobQueueManagerAPI} class. + */ + public static JobQueueManagerAPI getJobQueueManagerAPI() { + return (JobQueueManagerAPI) getInstance(APIIndex.JOB_QUEUE_MANAGER_API); + } + /** * Creates a single instance of the {@link CompanyAPI} class. * @@ -1339,7 +1349,8 @@ enum APIIndex CONTENT_TYPE_DESTROY_API, SYSTEM_API, ACHECKER_API, - CONTENT_ANALYTICS_API; + CONTENT_ANALYTICS_API, + JOB_QUEUE_MANAGER_API; Object create() { switch(this) { @@ -1433,6 +1444,7 @@ Object create() { case ARTIFICIAL_INTELLIGENCE_API: return new DotAIAPIFacadeImpl(); case ACHECKER_API: return new ACheckerAPIImpl(); case CONTENT_ANALYTICS_API: CDIUtils.getBean(ContentAnalyticsAPI.class).orElseThrow(() -> new DotRuntimeException("Content Analytics API not found")); + case JOB_QUEUE_MANAGER_API: return CDIUtils.getBean(JobQueueManagerAPI.class).orElseThrow(() -> new DotRuntimeException("JobQueueManagerAPI not found")); } throw new AssertionError("Unknown API index: " + this); } diff --git a/dotCMS/src/main/java/com/dotmarketing/util/ActivityLogger.java b/dotCMS/src/main/java/com/dotmarketing/util/ActivityLogger.java index 81fc4a9ed66f..ed2d1be0b1ec 100644 --- a/dotCMS/src/main/java/com/dotmarketing/util/ActivityLogger.java +++ b/dotCMS/src/main/java/com/dotmarketing/util/ActivityLogger.java @@ -31,7 +31,7 @@ public static void logDebug(Class cl, String action, String msg, String hostName } private static String getHostName(String hostNameOrId){ - if (!UtilMethods.isSet(hostNameOrId)) { + if (!UtilMethods.isSet(hostNameOrId) || "SYSTEM_HOST".equals(hostNameOrId)) { return "system"; } Host h = new Host(); diff --git a/dotCMS/src/main/resources/postgres.sql b/dotCMS/src/main/resources/postgres.sql index 700139aea8cf..943bfb730ed9 100644 --- a/dotCMS/src/main/resources/postgres.sql +++ b/dotCMS/src/main/resources/postgres.sql @@ -2523,3 +2523,52 @@ create table system_table ( -- Set up "like 'param%'" indexes for inode and identifier CREATE INDEX if not exists inode_inode_leading_idx ON inode(inode COLLATE "C"); CREATE INDEX if not exists identifier_id_leading_idx ON identifier(id COLLATE "C"); + +-- Table for active jobs in the queue +CREATE TABLE job_queue +( + id VARCHAR(255) PRIMARY KEY, + queue_name VARCHAR(255) NOT NULL, + state VARCHAR(50) NOT NULL, + priority INTEGER DEFAULT 0, + created_at timestamptz NOT NULL +); + +-- Table for job details and historical record +CREATE TABLE job +( + id VARCHAR(255) PRIMARY KEY, + queue_name VARCHAR(255) NOT NULL, + state VARCHAR(50) NOT NULL, + parameters JSONB NOT NULL, + result JSONB, + progress FLOAT DEFAULT 0, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + started_at timestamptz, + completed_at timestamptz, + execution_node VARCHAR(255), + retry_count INTEGER DEFAULT 0 +); + +-- Table for detailed job history +CREATE TABLE job_history +( + id VARCHAR(255) PRIMARY KEY, + job_id VARCHAR(255) NOT NULL, + state VARCHAR(50) NOT NULL, + execution_node VARCHAR(255), + created_at timestamptz NOT NULL, + result JSONB, + FOREIGN KEY (job_id) REFERENCES job (id) +); + +-- Indexes (add an index for the new parameters field in job_queue) +CREATE INDEX idx_job_queue_status ON job_queue (state); +CREATE INDEX idx_job_queue_priority_created_at ON job_queue (priority DESC, created_at ASC); +CREATE INDEX idx_job_parameters ON job USING GIN (parameters); +CREATE INDEX idx_job_result ON job USING GIN (result); +CREATE INDEX idx_job_status ON job (state); +CREATE INDEX idx_job_created_at ON job (created_at); +CREATE INDEX idx_job_history_job_id ON job_history (job_id); +CREATE INDEX idx_job_history_job_id_state ON job_history (job_id, state); diff --git a/dotcms-integration/src/test/java/com/dotcms/Junit5Suite1.java b/dotcms-integration/src/test/java/com/dotcms/Junit5Suite1.java index 2f5b5c15335f..ff80711f56d4 100644 --- a/dotcms-integration/src/test/java/com/dotcms/Junit5Suite1.java +++ b/dotcms-integration/src/test/java/com/dotcms/Junit5Suite1.java @@ -1,11 +1,17 @@ package com.dotcms; import com.dotcms.jobs.business.api.JobQueueManagerAPICDITest; +import com.dotcms.jobs.business.api.JobQueueManagerAPIIntegrationTest; +import com.dotcms.jobs.business.queue.PostgresJobQueueIntegrationTest; import org.junit.platform.suite.api.SelectClasses; import org.junit.platform.suite.api.Suite; @Suite -@SelectClasses({JobQueueManagerAPICDITest.class}) +@SelectClasses({ + JobQueueManagerAPICDITest.class, + PostgresJobQueueIntegrationTest.class, + JobQueueManagerAPIIntegrationTest.class +}) public class Junit5Suite1 { } diff --git a/dotcms-integration/src/test/java/com/dotcms/jobs/business/api/JobQueueManagerAPICDITest.java b/dotcms-integration/src/test/java/com/dotcms/jobs/business/api/JobQueueManagerAPICDITest.java index e015f0f37b10..bc028f524dd0 100644 --- a/dotcms-integration/src/test/java/com/dotcms/jobs/business/api/JobQueueManagerAPICDITest.java +++ b/dotcms-integration/src/test/java/com/dotcms/jobs/business/api/JobQueueManagerAPICDITest.java @@ -1,8 +1,12 @@ package com.dotcms.jobs.business.api; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertSame; +import com.dotcms.jobs.business.api.events.EventProducer; +import com.dotcms.jobs.business.api.events.RealTimeJobMonitor; import com.dotcms.jobs.business.error.CircuitBreaker; import com.dotcms.jobs.business.error.ExponentialBackoffRetryStrategy; import com.dotcms.jobs.business.error.RetryStrategy; @@ -14,7 +18,6 @@ import org.jboss.weld.junit5.WeldInitiator; import org.jboss.weld.junit5.WeldJunit5Extension; import org.jboss.weld.junit5.WeldSetup; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -32,7 +35,8 @@ public class JobQueueManagerAPICDITest { .beanClasses(JobQueueManagerAPIImpl.class, JobQueueConfig.class, JobQueue.class, RetryStrategy.class, CircuitBreaker.class, JobQueueProducer.class, JobQueueConfigProducer.class, - RetryStrategyProducer.class) + RetryStrategyProducer.class, RealTimeJobMonitor.class, + EventProducer.class) ); @Inject @@ -65,7 +69,7 @@ void test_SingletonBehavior() { @Test void test_CDIInjection() { assertNotNull(jobQueueManagerAPI, "JobQueueManagerAPI should be injected"); - Assertions.assertInstanceOf(JobQueueManagerAPIImpl.class, jobQueueManagerAPI, + assertInstanceOf(JobQueueManagerAPIImpl.class, jobQueueManagerAPI, "JobQueueManagerAPI should be an instance of JobQueueManagerAPIImpl"); } @@ -77,17 +81,18 @@ void test_CDIInjection() { @Test void test_JobQueueManagerAPIFields() { - // There are not JobQueue implementations yet - //Assertions.assertNotNull(impl.getJobQueue(), "JobQueue should be injected"); + assertNotNull(jobQueueManagerAPI.getJobQueue(), "JobQueue should be injected"); + assertInstanceOf(JobQueue.class, jobQueueManagerAPI.getJobQueue(), + "Injected object should implement JobQueue interface"); assertNotNull(jobQueueManagerAPI.getCircuitBreaker(), "CircuitBreaker should be injected"); assertNotNull(jobQueueManagerAPI.getDefaultRetryStrategy(), "Retry strategy should be injected"); - Assertions.assertEquals(10, jobQueueManagerAPI.getThreadPoolSize(), + assertEquals(10, jobQueueManagerAPI.getThreadPoolSize(), "ThreadPoolSize should be greater than 0"); - Assertions.assertInstanceOf(ExponentialBackoffRetryStrategy.class, + assertInstanceOf(ExponentialBackoffRetryStrategy.class, jobQueueManagerAPI.getDefaultRetryStrategy(), "Retry strategy should be an instance of ExponentialBackoffRetryStrategy"); } diff --git a/dotcms-integration/src/test/java/com/dotcms/jobs/business/api/JobQueueManagerAPIIntegrationTest.java b/dotcms-integration/src/test/java/com/dotcms/jobs/business/api/JobQueueManagerAPIIntegrationTest.java new file mode 100644 index 000000000000..56a6f9e852d4 --- /dev/null +++ b/dotcms-integration/src/test/java/com/dotcms/jobs/business/api/JobQueueManagerAPIIntegrationTest.java @@ -0,0 +1,549 @@ +package com.dotcms.jobs.business.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.dotcms.jobs.business.error.ExponentialBackoffRetryStrategy; +import com.dotcms.jobs.business.error.RetryStrategy; +import com.dotcms.jobs.business.job.Job; +import com.dotcms.jobs.business.job.JobState; +import com.dotcms.jobs.business.processor.Cancellable; +import com.dotcms.jobs.business.processor.JobProcessor; +import com.dotcms.jobs.business.processor.ProgressTracker; +import com.dotcms.util.IntegrationTestInitService; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.common.db.DotConnect; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.util.Logger; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for the JobQueueManagerAPI. + * These tests verify the functionality of the job queue system in a real environment, + * including job creation, processing, cancellation, retrying, and progress tracking. + */ +public class JobQueueManagerAPIIntegrationTest { + + private static JobQueueManagerAPI jobQueueManagerAPI; + + /** + * Sets up the test environment before all tests are run. + * Initializes the test environment and obtains an instance of JobQueueManagerAPI. + * + * @throws Exception if there's an error during setup + */ + @BeforeAll + static void setUp() throws Exception { + + // Initialize the test environment + IntegrationTestInitService.getInstance().init(); + + jobQueueManagerAPI = APILocator.getJobQueueManagerAPI(); + } + + /** + * Cleans up the test environment after all tests have run. + * Closes the JobQueueManagerAPI and clears all jobs from the database. + * + * @throws Exception if there's an error during cleanup + */ + @AfterAll + static void cleanUp() throws Exception { + + jobQueueManagerAPI.close(); + clearJobs(); + } + + @BeforeEach + void reset() { + // Reset circuit breaker + jobQueueManagerAPI.getCircuitBreaker().reset(); + } + + /** + * Method to test: createJob and process execution in JobQueueManagerAPI + * Given Scenario: A job is created and submitted to the queue + * ExpectedResult: The job is successfully created, processed, and completed within the expected timeframe + */ + @Test + void test_CreateAndProcessJob() throws Exception { + + // Register a test processor + jobQueueManagerAPI.registerProcessor("testQueue", new TestJobProcessor()); + + // Start the JobQueueManagerAPI + if (!jobQueueManagerAPI.isStarted()) { + jobQueueManagerAPI.start(); + jobQueueManagerAPI.awaitStart(5, TimeUnit.SECONDS); + } + + // Create a job + Map parameters = new HashMap<>(); + parameters.put("testParam", "testValue"); + String jobId = jobQueueManagerAPI.createJob("testQueue", parameters); + + assertNotNull(jobId, "Job ID should not be null"); + + // Wait for the job to be processed + CountDownLatch latch = new CountDownLatch(1); + jobQueueManagerAPI.watchJob(jobId, job -> { + if (job.state() == JobState.COMPLETED) { + latch.countDown(); + } + }); + + boolean processed = latch.await(10, TimeUnit.SECONDS); + assertTrue(processed, "Job should be processed within 10 seconds"); + + // Wait for job processing to complete + Awaitility.await().atMost(10, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + Job job = jobQueueManagerAPI.getJob(jobId); + assertEquals(JobState.COMPLETED, job.state(), + "Job should be in COMPLETED state"); + }); + } + + /** + * Method to test: Job failure handling in JobQueueManagerAPI + * Given Scenario: A job is created that is designed to fail + * ExpectedResult: The job fails, is marked as FAILED, and contains the expected error details + */ + @Test + void test_FailingJob() throws Exception { + + jobQueueManagerAPI.registerProcessor("failingQueue", new FailingJobProcessor()); + RetryStrategy contentImportRetryStrategy = new ExponentialBackoffRetryStrategy( + 5000, 300000, 2.0, 0 + ); + jobQueueManagerAPI.setRetryStrategy("failingQueue", contentImportRetryStrategy); + + if (!jobQueueManagerAPI.isStarted()) { + jobQueueManagerAPI.start(); + jobQueueManagerAPI.awaitStart(5, TimeUnit.SECONDS); + } + + Map parameters = new HashMap<>(); + String jobId = jobQueueManagerAPI.createJob("failingQueue", parameters); + + CountDownLatch latch = new CountDownLatch(1); + jobQueueManagerAPI.watchJob(jobId, job -> { + if (job.state() == JobState.FAILED) { + latch.countDown(); + } + }); + + boolean processed = latch.await(10, TimeUnit.SECONDS); + assertTrue(processed, "Job should be processed (and fail) within 10 seconds"); + + // Wait for job processing to complete + Awaitility.await().atMost(10, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + Job job = jobQueueManagerAPI.getJob(jobId); + assertEquals(JobState.FAILED, job.state(), + "Job should be in FAILED state"); + assertNotNull(job.result().get().errorDetail().get(), + "Job should have an error detail"); + assertEquals("Simulated failure", + job.result().get().errorDetail().get().message(), + "Error message should match"); + }); + } + + /** + * Method to test: cancelJob method in JobQueueManagerAPI + * Given Scenario: A running job is canceled + * ExpectedResult: The job is successfully canceled, its state is set to CANCELED, and the + * processor acknowledges the cancellation + */ + @Test + void test_CancelJob() throws Exception { + + CancellableJobProcessor processor = new CancellableJobProcessor(); + jobQueueManagerAPI.registerProcessor("cancellableQueue", processor); + + if (!jobQueueManagerAPI.isStarted()) { + jobQueueManagerAPI.start(); + jobQueueManagerAPI.awaitStart(5, TimeUnit.SECONDS); + } + + Map parameters = new HashMap<>(); + String jobId = jobQueueManagerAPI.createJob("cancellableQueue", parameters); + + Awaitility.await().atMost(5, TimeUnit.SECONDS) + .until(() -> { + Job job = jobQueueManagerAPI.getJob(jobId); + return job.state() == JobState.RUNNING; + }); + + // Cancel the job + jobQueueManagerAPI.cancelJob(jobId); + + CountDownLatch latch = new CountDownLatch(1); + jobQueueManagerAPI.watchJob(jobId, job -> { + if (job.state() == JobState.CANCELED) { + latch.countDown(); + } + }); + + boolean processed = latch.await(10, TimeUnit.SECONDS); + assertTrue(processed, "Job should be canceled within 10 seconds"); + + // Wait for job processing to complete + Awaitility.await().atMost(10, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + Job job = jobQueueManagerAPI.getJob(jobId); + assertEquals(JobState.CANCELED, job.state(), + "Job should be in CANCELED state"); + assertTrue(processor.wasCanceled(), + "Job processor should have been canceled"); + }); + } + + /** + * Method to test: Job retry mechanism in JobQueueManagerAPI + * Given Scenario: A job is created that fails initially but succeeds after a certain number + * of retries + * ExpectedResult: The job is retried the configured number of times, eventually succeeds, and + * is marked as COMPLETED + */ + @Test + void test_JobRetry() throws Exception { + + int maxRetries = 3; + RetryingJobProcessor processor = new RetryingJobProcessor(maxRetries); + jobQueueManagerAPI.registerProcessor("retryQueue", processor); + + RetryStrategy retryStrategy = new ExponentialBackoffRetryStrategy( + 100, 1000, 2.0, maxRetries + ); + jobQueueManagerAPI.setRetryStrategy("retryQueue", retryStrategy); + + if (!jobQueueManagerAPI.isStarted()) { + jobQueueManagerAPI.start(); + jobQueueManagerAPI.awaitStart(5, TimeUnit.SECONDS); + } + + Map parameters = new HashMap<>(); + String jobId = jobQueueManagerAPI.createJob("retryQueue", parameters); + + CountDownLatch latch = new CountDownLatch(1); + jobQueueManagerAPI.watchJob(jobId, job -> { + if (job.state() == JobState.COMPLETED) { + latch.countDown(); + } + }); + + boolean processed = latch.await(30, TimeUnit.SECONDS); + assertTrue(processed, "Job should be processed within 30 seconds"); + + // Wait for job processing to complete + Awaitility.await().atMost(30, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + Job job = jobQueueManagerAPI.getJob(jobId); + assertEquals(JobState.COMPLETED, job.state(), + "Job should be in COMPLETED state"); + assertEquals(maxRetries + 1, processor.getAttempts(), + "Job should have been attempted " + maxRetries + " times"); + }); + } + + /** + * Method to test: Progress tracking functionality in JobQueueManagerAPI + * Given Scenario: A job is created that reports progress during its execution + * ExpectedResult: Progress updates are received, increase monotonically, and the job completes + * with 100% progress + */ + @Test + void test_JobWithProgressTracker() throws Exception { + + // Register a processor that uses progress tracking + ProgressTrackingJobProcessor processor = new ProgressTrackingJobProcessor(); + jobQueueManagerAPI.registerProcessor("progressQueue", processor); + + // Start the JobQueueManagerAPI + if (!jobQueueManagerAPI.isStarted()) { + jobQueueManagerAPI.start(); + jobQueueManagerAPI.awaitStart(5, TimeUnit.SECONDS); + } + + // Create a job + Map parameters = new HashMap<>(); + String jobId = jobQueueManagerAPI.createJob("progressQueue", parameters); + + List progressUpdates = Collections.synchronizedList(new ArrayList<>()); + + // Watch the job and collect progress updates + jobQueueManagerAPI.watchJob(jobId, job -> { + progressUpdates.add(job.progress()); + }); + + // Wait for the job to complete + Awaitility.await().atMost(30, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .until(() -> { + Job job = jobQueueManagerAPI.getJob(jobId); + return job.state() == JobState.COMPLETED; + }); + + // Verify final job state + Job completedJob = jobQueueManagerAPI.getJob(jobId); + assertEquals(JobState.COMPLETED, completedJob.state(), "Job should be in COMPLETED state"); + assertEquals(1.0f, completedJob.progress(), 0.01f, "Final progress should be 1.0"); + + // Verify progress updates + assertFalse(progressUpdates.isEmpty(), "Should have received progress updates"); + assertEquals(0.0f, progressUpdates.get(0), 0.01f, "Initial progress should be 0"); + assertEquals(1.0f, progressUpdates.get(progressUpdates.size() - 1), 0.01f, "Final progress should be 1"); + + // Verify that progress increased monotonically + for (int i = 1; i < progressUpdates.size(); i++) { + assertTrue(progressUpdates.get(i) >= progressUpdates.get(i - 1), + "Progress should increase or stay the same"); + } + } + + /** + * Method to test: Multiple scenarios in JobQueueManagerAPI including success, failure, and + * cancellation Given Scenario: Multiple jobs are created simultaneously with different expected + * outcomes: + * - Two jobs expected to succeed + * - One job expected to fail and be retried + * - One job to be canceled mid-execution ExpectedResult: All jobs reach their expected final + * states within the timeout period: + * - Successful jobs complete + * - Failing job retries and ultimately fails + * - Cancellable job is successfully canceled All job states and related details (retry counts, + * error details, cancellation status) are verified + */ + @Test + void test_CombinedScenarios() throws Exception { + + // Register processors for different scenarios + jobQueueManagerAPI.registerProcessor("successQueue", new TestJobProcessor()); + jobQueueManagerAPI.registerProcessor("failQueue", new FailingJobProcessor()); + jobQueueManagerAPI.registerProcessor("cancelQueue", new CancellableJobProcessor()); + + // Set up retry strategy for failing jobs + RetryStrategy retryStrategy = new ExponentialBackoffRetryStrategy( + 100, 1000, 2.0, 2 + ); + jobQueueManagerAPI.setRetryStrategy("failQueue", retryStrategy); + + // Ensure JobQueueManagerAPI is started + if (!jobQueueManagerAPI.isStarted()) { + jobQueueManagerAPI.start(); + jobQueueManagerAPI.awaitStart(5, TimeUnit.SECONDS); + } + + // Create jobs + String successJob1Id = jobQueueManagerAPI.createJob("successQueue", new HashMap<>()); + String successJob2Id = jobQueueManagerAPI.createJob("successQueue", new HashMap<>()); + String failJobId = jobQueueManagerAPI.createJob("failQueue", new HashMap<>()); + String cancelJobId = jobQueueManagerAPI.createJob("cancelQueue", new HashMap<>()); + + // Set up latches to track job completions + CountDownLatch successLatch = new CountDownLatch(2); + CountDownLatch failLatch = new CountDownLatch(1); + CountDownLatch cancelLatch = new CountDownLatch(1); + + // Watch jobs + jobQueueManagerAPI.watchJob(successJob1Id, job -> { + if (job.state() == JobState.COMPLETED) { + successLatch.countDown(); + } + }); + jobQueueManagerAPI.watchJob(successJob2Id, job -> { + if (job.state() == JobState.COMPLETED) { + successLatch.countDown(); + } + }); + jobQueueManagerAPI.watchJob(failJobId, job -> { + if (job.state() == JobState.FAILED) { + failLatch.countDown(); + } + }); + jobQueueManagerAPI.watchJob(cancelJobId, job -> { + if (job.state() == JobState.CANCELED) { + cancelLatch.countDown(); + } + }); + + // Wait a bit before cancelling the job + Awaitility.await().pollDelay(500, TimeUnit.MILLISECONDS).until(() -> true); + jobQueueManagerAPI.cancelJob(cancelJobId); + + // Wait for all jobs to complete (or timeout after 30 seconds) + boolean allCompleted = successLatch.await(30, TimeUnit.SECONDS) + && failLatch.await(30, TimeUnit.SECONDS) + && cancelLatch.await(30, TimeUnit.SECONDS); + + assertTrue(allCompleted, "All jobs should complete within the timeout period"); + + // Verify final states + assertEquals(JobState.COMPLETED, jobQueueManagerAPI.getJob(successJob1Id).state(), + "First success job should be completed"); + assertEquals(JobState.COMPLETED, jobQueueManagerAPI.getJob(successJob2Id).state(), + "Second success job should be completed"); + assertEquals(JobState.FAILED, jobQueueManagerAPI.getJob(failJobId).state(), + "Fail job should be in failed state"); + assertEquals(JobState.CANCELED, jobQueueManagerAPI.getJob(cancelJobId).state(), + "Cancel job should be canceled"); + + // Wait for job processing to complete as we have retries running + Awaitility.await().atMost(30, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + Job failedJob = jobQueueManagerAPI.getJob(failJobId); + assertEquals(2, failedJob.retryCount(), + "Job should have been retried " + 2 + " times"); + assertEquals(JobState.FAILED, failedJob.state(), + "Job should be in FAILED state"); + assertTrue(failedJob.result().isPresent(), + "Failed job should have a result"); + assertTrue(failedJob.result().get().errorDetail().isPresent(), + "Failed job should have error details"); + }); + } + + private static class ProgressTrackingJobProcessor implements JobProcessor { + @Override + public void process(Job job) { + ProgressTracker tracker = job.progressTracker().orElseThrow( + () -> new IllegalStateException("Progress tracker not set") + ); + for (int i = 0; i <= 10; i++) { + float progress = i / 10.0f; + tracker.updateProgress(progress); + + // Simulate work being done + Awaitility.await().pollDelay(500, TimeUnit.MILLISECONDS).until(() -> true); + } + } + + @Override + public Map getResultMetadata(Job job) { + return new HashMap<>(); + } + } + + private static class RetryingJobProcessor implements JobProcessor { + + private final int maxRetries; + private int attempts = 0; + + public RetryingJobProcessor(int maxRetries) { + this.maxRetries = maxRetries; + } + + @Override + public void process(Job job) { + attempts++; + if (attempts <= maxRetries) { + throw new RuntimeException("Simulated failure, attempt " + attempts); + } + // If we've reached here, we've exceeded maxRetries and the job should succeed + } + + @Override + public Map getResultMetadata(Job job) { + Map metadata = new HashMap<>(); + metadata.put("attempts", attempts); + return metadata; + } + + public int getAttempts() { + return attempts; + } + } + + private static class FailingJobProcessor implements JobProcessor { + + @Override + public void process(Job job) { + throw new RuntimeException("Simulated failure"); + } + + @Override + public Map getResultMetadata(Job job) { + return new HashMap<>(); + } + } + + private static class CancellableJobProcessor implements JobProcessor, Cancellable { + + private final AtomicBoolean canceled = new AtomicBoolean(false); + private final AtomicBoolean wasCanceled = new AtomicBoolean(false); + + @Override + public void process(Job job) { + + Awaitility.await().atMost(10, TimeUnit.SECONDS) + .until(canceled::get); + + // Simulate some additional work after cancellation + Awaitility.await().pollDelay(1, TimeUnit.SECONDS).until(() -> true); + } + + @Override + public void cancel(Job job) { + canceled.set(true); + wasCanceled.set(true); + } + + @Override + public Map getResultMetadata(Job job) { + return new HashMap<>(); + } + + public boolean wasCanceled() { + return wasCanceled.get(); + } + } + + private static class TestJobProcessor implements JobProcessor { + + @Override + public void process(Job job) { + // Simulate some work + Awaitility.await().pollDelay(1, TimeUnit.SECONDS).until(() -> true); + } + + @Override + public Map getResultMetadata(Job job) { + return new HashMap<>(); + } + } + + /** + * Helper method to clear all jobs from the database + */ + private static void clearJobs() { + try { + new DotConnect().setSQL("delete from job_history").loadResult(); + new DotConnect().setSQL("delete from job_queue").loadResult(); + new DotConnect().setSQL("delete from job").loadResult(); + } catch (DotDataException e) { + Logger.warn(JobQueueManagerAPIIntegrationTest.class, "Error cleaning up jobs", e); + } + } + +} \ No newline at end of file diff --git a/dotcms-integration/src/test/java/com/dotcms/jobs/business/api/JobQueueManagerAPITest.java b/dotcms-integration/src/test/java/com/dotcms/jobs/business/api/JobQueueManagerAPITest.java index 212f61ffc302..72fdfb7188d0 100644 --- a/dotcms-integration/src/test/java/com/dotcms/jobs/business/api/JobQueueManagerAPITest.java +++ b/dotcms-integration/src/test/java/com/dotcms/jobs/business/api/JobQueueManagerAPITest.java @@ -5,7 +5,9 @@ import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.anyFloat; +import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyMap; import static org.mockito.Mockito.anyString; @@ -20,30 +22,43 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.dotcms.jobs.business.api.events.EventProducer; +import com.dotcms.jobs.business.api.events.RealTimeJobMonitor; import com.dotcms.jobs.business.error.CircuitBreaker; import com.dotcms.jobs.business.error.ErrorDetail; +import com.dotcms.jobs.business.error.JobCancellationException; import com.dotcms.jobs.business.error.JobProcessingException; import com.dotcms.jobs.business.error.RetryStrategy; import com.dotcms.jobs.business.job.Job; +import com.dotcms.jobs.business.job.JobPaginatedResult; +import com.dotcms.jobs.business.job.JobResult; import com.dotcms.jobs.business.job.JobState; +import com.dotcms.jobs.business.processor.Cancellable; import com.dotcms.jobs.business.processor.DefaultProgressTracker; import com.dotcms.jobs.business.processor.JobProcessor; import com.dotcms.jobs.business.processor.ProgressTracker; import com.dotcms.jobs.business.queue.JobQueue; +import com.dotcms.jobs.business.queue.error.JobLockingException; +import com.dotcms.jobs.business.queue.error.JobNotFoundException; +import com.dotcms.jobs.business.queue.error.JobQueueDataException; +import com.dotcms.jobs.business.queue.error.JobQueueException; +import com.dotmarketing.exception.DotDataException; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import javax.enterprise.event.Event; import org.awaitility.Awaitility; import org.junit.Before; import org.junit.Test; @@ -62,6 +77,8 @@ public class JobQueueManagerAPITest { private JobQueueManagerAPI jobQueueManagerAPI; + private EventProducer eventProducer; + @Before public void setUp() { @@ -69,12 +86,18 @@ public void setUp() { mockJobProcessor = mock(JobProcessor.class); mockRetryStrategy = mock(RetryStrategy.class); mockCircuitBreaker = mock(CircuitBreaker.class); + eventProducer = mock(EventProducer.class); - jobQueueManagerAPI = new JobQueueManagerAPIImpl( - mockJobQueue, new JobQueueConfig(1), mockCircuitBreaker, mockRetryStrategy + jobQueueManagerAPI = newJobQueueManagerAPI( + mockJobQueue, mockCircuitBreaker, mockRetryStrategy, eventProducer, + 1, 10 ); + jobQueueManagerAPI.registerProcessor("testQueue", mockJobProcessor); jobQueueManagerAPI.setRetryStrategy("testQueue", mockRetryStrategy); + + var event = mock(Event.class); + when(eventProducer.getEvent(any())).thenReturn(event); } /** @@ -83,16 +106,16 @@ mockJobQueue, new JobQueueConfig(1), mockCircuitBreaker, mockRetryStrategy * ExpectedResult: Job is created successfully and correct job ID is returned */ @Test - public void test_createJob() { + public void test_createJob() throws DotDataException, JobQueueException { Map parameters = new HashMap<>(); - when(mockJobQueue.addJob(anyString(), anyMap())).thenReturn("job123"); + when(mockJobQueue.createJob(anyString(), anyMap())).thenReturn("job123"); // Creating a job String jobId = jobQueueManagerAPI.createJob("testQueue", parameters); assertEquals("job123", jobId); - verify(mockJobQueue).addJob("testQueue", parameters); + verify(mockJobQueue).createJob("testQueue", parameters); } /** @@ -101,7 +124,7 @@ public void test_createJob() { * ExpectedResult: Correct job is retrieved from the job queue */ @Test - public void test_getJob() { + public void test_getJob() throws DotDataException, JobQueueDataException, JobNotFoundException { Job mockJob = mock(Job.class); when(mockJobQueue.getJob("job123")).thenReturn(mockJob); @@ -119,21 +142,27 @@ public void test_getJob() { * ExpectedResult: Correct list of jobs is retrieved from the job queue */ @Test - public void test_getJobs() { + public void test_getJobs() throws DotDataException, JobQueueDataException { // Prepare test data Job job1 = mock(Job.class); Job job2 = mock(Job.class); List expectedJobs = Arrays.asList(job1, job2); + final var paginatedResult = JobPaginatedResult.builder() + .jobs(expectedJobs) + .total(2) + .page(1) + .pageSize(10) + .build(); // Mock the behavior of jobQueue.getJobs - when(mockJobQueue.getJobs(1, 10)).thenReturn(expectedJobs); + when(mockJobQueue.getJobs(1, 10)).thenReturn(paginatedResult); // Call the method under test - List actualJobs = jobQueueManagerAPI.getJobs(1, 10); + final var actualResult = jobQueueManagerAPI.getJobs(1, 10); // Verify the results - assertEquals(expectedJobs, actualJobs); + assertEquals(expectedJobs, actualResult.jobs()); verify(mockJobQueue).getJobs(1, 10); } @@ -143,7 +172,8 @@ public void test_getJobs() { * ExpectedResult: JobQueueManagerAPI starts successfully and begins processing jobs */ @Test - public void test_start() throws InterruptedException { + public void test_start() + throws InterruptedException, JobQueueDataException, JobLockingException { // Make the circuit breaker always allow requests when(mockCircuitBreaker.allowRequest()).thenReturn(true); @@ -225,42 +255,48 @@ public void test_close() throws Exception { @Test public void test_watchJob() throws Exception { - // Prepare test data - String jobId = "testJobId"; - Job initialJob = mock(Job.class); - when(initialJob.id()).thenReturn(jobId); - when(initialJob.queueName()).thenReturn("testQueue"); - when(initialJob.state()).thenReturn(JobState.PENDING); - - // Mock behavior for job state changes - Job runningJob = mock(Job.class); - when(runningJob.id()).thenReturn(jobId); - when(runningJob.queueName()).thenReturn("testQueue"); - when(runningJob.state()).thenReturn(JobState.RUNNING); - when(initialJob.withState(JobState.RUNNING)).thenReturn(runningJob); - - Job completedJob = mock(Job.class); - when(completedJob.id()).thenReturn(jobId); - when(completedJob.queueName()).thenReturn("testQueue"); - when(completedJob.state()).thenReturn(JobState.COMPLETED); - when(runningJob.markAsCompleted()).thenReturn(completedJob); + // Create a mock job + String jobId = "job123"; + Job mockJob = mock(Job.class); + when(mockJob.id()).thenReturn(jobId); + when(mockJob.queueName()).thenReturn("testQueue"); // Mock JobQueue behavior - when(mockJobQueue.getJob(jobId)).thenReturn(initialJob); - when(mockJobQueue.nextJob()).thenReturn(initialJob).thenReturn(null); + when(mockJobQueue.getJob(jobId)).thenReturn(mockJob); + when(mockJobQueue.nextJob()).thenReturn(mockJob).thenReturn(null); + when(mockJob.markAsRunning()).thenReturn(mockJob); + when(mockJob.withProgressTracker(any(DefaultProgressTracker.class))).thenReturn(mockJob); // Make the circuit breaker always allow requests when(mockCircuitBreaker.allowRequest()).thenReturn(true); // Mock JobProcessor behavior ProgressTracker mockProgressTracker = mock(ProgressTracker.class); - when(mockJobProcessor.progressTracker(any())).thenReturn(mockProgressTracker); - when(runningJob.progress()).thenReturn(0f); - when(runningJob.withProgress(anyFloat())).thenReturn(runningJob); + when(mockJob.progressTracker()).thenReturn(Optional.ofNullable(mockProgressTracker)); + when(mockJob.progress()).thenReturn(0f); + when(mockJob.withProgress(anyFloat())).thenReturn(mockJob); + + AtomicReference jobState = new AtomicReference<>(JobState.PENDING); + when(mockJob.state()).thenAnswer(inv -> jobState.get()); + + when(mockJob.withState(any())).thenAnswer(inv -> { + jobState.set(inv.getArgument(0)); + return mockJob; + }); + when(mockJob.markAsCompleted(any())).thenAnswer(inv -> { + jobState.set(JobState.COMPLETED); + return mockJob; + }); + when(mockJob.markAsFailed(any())).thenAnswer(inv -> { + jobState.set(JobState.FAILED); + return mockJob; + }); + + when(mockJobQueue.getUpdatedJobsSince(anySet(), any(LocalDateTime.class))) + .thenAnswer(invocation -> Collections.singletonList(mockJob)); // Create a list to capture job states List capturedStates = Collections.synchronizedList(new ArrayList<>()); - // Create a test watcher Consumer testWatcher = job -> { assertNotNull(job); @@ -280,16 +316,6 @@ public void test_watchJob() throws Exception { .pollInterval(100, TimeUnit.MILLISECONDS) .until(() -> capturedStates.contains(JobState.COMPLETED)); - // Verify job processing - verify(mockJobQueue, timeout(5000)).updateJobStatus(runningJob); - verify(mockJobProcessor, timeout(5000)).process(runningJob); - verify(mockJobQueue, timeout(5000)).updateJobStatus(completedJob); - - // Verify watcher received all job states - assertTrue(capturedStates.contains(JobState.PENDING)); - assertTrue(capturedStates.contains(JobState.RUNNING)); - assertTrue(capturedStates.contains(JobState.COMPLETED)); - // Stop the JobQueueManagerAPI jobQueueManagerAPI.close(); } @@ -313,15 +339,21 @@ public void test_JobRetry_single_retry() throws Exception { AtomicInteger retryCount = new AtomicInteger(0); when(mockJob.retryCount()).thenAnswer(inv -> retryCount.get()); + when(mockJob.completedAt()).thenAnswer(inv -> Optional.of(LocalDateTime.now())); + when(mockJob.withState(any())).thenAnswer(inv -> { jobState.set(inv.getArgument(0)); return mockJob; }); + when(mockJob.markAsRunning()).thenAnswer(inv -> { + jobState.set(JobState.RUNNING); + return mockJob; + }); when(mockJob.incrementRetry()).thenAnswer(inv -> { retryCount.incrementAndGet(); return mockJob; }); - when(mockJob.markAsCompleted()).thenAnswer(inv -> { + when(mockJob.markAsCompleted(any())).thenAnswer(inv -> { jobState.set(JobState.COMPLETED); return mockJob; }); @@ -330,6 +362,8 @@ public void test_JobRetry_single_retry() throws Exception { return mockJob; }); + when(mockJob.withProgressTracker(any(DefaultProgressTracker.class))).thenReturn(mockJob); + // Set up the job queue to return our mock job twice (for initial attempt and retry) when(mockJobQueue.nextJob()).thenReturn(mockJob, mockJob, null); @@ -342,7 +376,7 @@ public void test_JobRetry_single_retry() throws Exception { // Configure progress tracker ProgressTracker mockProgressTracker = mock(ProgressTracker.class); - when(mockJobProcessor.progressTracker(any())).thenReturn(mockProgressTracker); + when(mockJob.progressTracker()).thenReturn(Optional.ofNullable(mockProgressTracker)); when(mockJob.progress()).thenReturn(0f); when(mockJob.withProgress(anyFloat())).thenReturn(mockJob); @@ -353,7 +387,7 @@ public void test_JobRetry_single_retry() throws Exception { throw new RuntimeException("Simulated failure"); } Job job = invocation.getArgument(0); - job.markAsCompleted(); + job.markAsCompleted(any()); return null; }).when(mockJobProcessor).process(any()); @@ -395,22 +429,26 @@ public void test_JobRetry_retry_twice() throws Exception { AtomicReference jobState = new AtomicReference<>(JobState.PENDING); AtomicInteger retryCount = new AtomicInteger(0); - AtomicLong lastRetryTimestamp = new AtomicLong(System.currentTimeMillis()); + AtomicReference lastRetry = new AtomicReference<>(LocalDateTime.now()); when(mockJob.state()).thenAnswer(inv -> jobState.get()); when(mockJob.retryCount()).thenAnswer(inv -> retryCount.get()); - when(mockJob.lastRetryTimestamp()).thenAnswer(inv -> lastRetryTimestamp.get()); + when(mockJob.completedAt()).thenAnswer(inv -> Optional.of(lastRetry.get())); when(mockJob.withState(any())).thenAnswer(inv -> { jobState.set(inv.getArgument(0)); return mockJob; }); + when(mockJob.markAsRunning()).thenAnswer(inv -> { + jobState.set(JobState.RUNNING); + return mockJob; + }); when(mockJob.incrementRetry()).thenAnswer(inv -> { retryCount.incrementAndGet(); - lastRetryTimestamp.set(System.currentTimeMillis()); + lastRetry.set(LocalDateTime.now()); return mockJob; }); - when(mockJob.markAsCompleted()).thenAnswer(inv -> { + when(mockJob.markAsCompleted(any())).thenAnswer(inv -> { jobState.set(JobState.COMPLETED); return mockJob; }); @@ -419,6 +457,8 @@ public void test_JobRetry_retry_twice() throws Exception { return mockJob; }); + when(mockJob.withProgressTracker(any(DefaultProgressTracker.class))).thenReturn(mockJob); + // Configure job queue to always return the mockJob until it's completed when(mockJobQueue.nextJob()).thenAnswer(inv -> jobState.get() != JobState.COMPLETED ? mockJob : null @@ -433,7 +473,7 @@ public void test_JobRetry_retry_twice() throws Exception { // Configure progress tracker ProgressTracker mockProgressTracker = mock(ProgressTracker.class); - when(mockJobProcessor.progressTracker(any())).thenReturn(mockProgressTracker); + when(mockJob.progressTracker()).thenReturn(Optional.ofNullable(mockProgressTracker)); when(mockJob.progress()).thenReturn(0f); when(mockJob.withProgress(anyFloat())).thenReturn(mockJob); @@ -444,8 +484,6 @@ public void test_JobRetry_retry_twice() throws Exception { if (attempt < 2) { throw new RuntimeException("Simulated failure"); } - Job job = invocation.getArgument(0); - job.markAsCompleted(); return null; }).when(mockJobProcessor).process(any()); @@ -463,12 +501,12 @@ public void test_JobRetry_retry_twice() throws Exception { // Verify state transitions InOrder inOrder = inOrder(mockJob); - inOrder.verify(mockJob).withState(JobState.RUNNING); + inOrder.verify(mockJob).markAsRunning(); inOrder.verify(mockJob).markAsFailed(any()); - inOrder.verify(mockJob).withState(JobState.RUNNING); + inOrder.verify(mockJob).markAsRunning(); inOrder.verify(mockJob).markAsFailed(any()); - inOrder.verify(mockJob).withState(JobState.RUNNING); - inOrder.verify(mockJob).markAsCompleted(); + inOrder.verify(mockJob).markAsRunning(); + inOrder.verify(mockJob).markAsCompleted(any()); // Verify retry behavior verify(mockRetryStrategy, atLeast(2)).shouldRetry(any(), any()); @@ -497,20 +535,24 @@ public void test_JobRetry_MaxRetryLimit() throws Exception { AtomicReference jobState = new AtomicReference<>(JobState.PENDING); AtomicInteger retryCount = new AtomicInteger(0); - AtomicLong lastRetryTimestamp = new AtomicLong(System.currentTimeMillis()); + AtomicReference lastRetry = new AtomicReference<>(LocalDateTime.now()); int maxRetries = 3; when(mockJob.state()).thenAnswer(inv -> jobState.get()); when(mockJob.retryCount()).thenAnswer(inv -> retryCount.get()); - when(mockJob.lastRetryTimestamp()).thenAnswer(inv -> lastRetryTimestamp.get()); + when(mockJob.completedAt()).thenAnswer(inv -> Optional.of(lastRetry.get())); when(mockJob.withState(any())).thenAnswer(inv -> { jobState.set(inv.getArgument(0)); return mockJob; }); + when(mockJob.markAsRunning()).thenAnswer(inv -> { + jobState.set(JobState.RUNNING); + return mockJob; + }); when(mockJob.incrementRetry()).thenAnswer(inv -> { retryCount.incrementAndGet(); - lastRetryTimestamp.set(System.currentTimeMillis()); + lastRetry.set(LocalDateTime.now()); return mockJob; }); when(mockJob.markAsFailed(any())).thenAnswer(inv -> { @@ -518,6 +560,8 @@ public void test_JobRetry_MaxRetryLimit() throws Exception { return mockJob; }); + when(mockJob.withProgressTracker(any(DefaultProgressTracker.class))).thenReturn(mockJob); + // Configure job queue when(mockJobQueue.nextJob()).thenReturn(mockJob, mockJob, mockJob, mockJob, mockJob, null); @@ -531,7 +575,7 @@ public void test_JobRetry_MaxRetryLimit() throws Exception { // Configure progress tracker ProgressTracker mockProgressTracker = mock(ProgressTracker.class); - when(mockJobProcessor.progressTracker(any())).thenReturn(mockProgressTracker); + when(mockJob.progressTracker()).thenReturn(Optional.ofNullable(mockProgressTracker)); // Configure job processor to always fail doThrow(new RuntimeException("Simulated failure")).when(mockJobProcessor).process(any()); @@ -552,7 +596,7 @@ public void test_JobRetry_MaxRetryLimit() throws Exception { // Verify the job was not retried after reaching the max retry limit verify(mockRetryStrategy, times(maxRetries + 1)). shouldRetry(any(), any()); // Retries + final attempt - verify(mockJobQueue, times(1)).removeJob(mockJob.id()); + verify(mockJobQueue, times(1)).removeJobFromQueue(mockJob.id()); // Stop the job queue jobQueueManagerAPI.close(); @@ -581,24 +625,29 @@ public void test_Job_SucceedsFirstAttempt() throws Exception { jobState.set(inv.getArgument(0)); return mockJob; }); - when(mockJob.markAsCompleted()).thenAnswer(inv -> { + when(mockJob.markAsRunning()).thenAnswer(inv -> { + jobState.set(JobState.RUNNING); + return mockJob; + }); + when(mockJob.markAsCompleted(any())).thenAnswer(inv -> { jobState.set(JobState.COMPLETED); return mockJob; }); + when(mockJob.withProgressTracker(any(DefaultProgressTracker.class))).thenReturn(mockJob); // Configure job queue when(mockJobQueue.nextJob()).thenReturn(mockJob, null); // Configure progress tracker ProgressTracker mockProgressTracker = mock(ProgressTracker.class); - when(mockJobProcessor.progressTracker(any())).thenReturn(mockProgressTracker); + when(mockJob.progressTracker()).thenReturn(Optional.ofNullable(mockProgressTracker)); when(mockJob.progress()).thenReturn(0f); when(mockJob.withProgress(anyFloat())).thenReturn(mockJob); // Configure job processor to succeed doAnswer(inv -> { Job job = inv.getArgument(0); - job.markAsCompleted(); + job.markAsCompleted(any()); return null; }).when(mockJobProcessor).process(any()); @@ -643,10 +692,15 @@ public void test_Job_NotRetryable() throws Exception { jobState.set(inv.getArgument(0)); return mockJob; }); + when(mockJob.markAsRunning()).thenAnswer(inv -> { + jobState.set(JobState.RUNNING); + return mockJob; + }); when(mockJob.markAsFailed(any())).thenAnswer(inv -> { jobState.set(JobState.FAILED); return mockJob; }); + when(mockJob.withProgressTracker(any(DefaultProgressTracker.class))).thenReturn(mockJob); // Configure job queue when(mockJobQueue.nextJob()).thenReturn(mockJob, mockJob, null); @@ -659,7 +713,7 @@ public void test_Job_NotRetryable() throws Exception { // Configure progress tracker ProgressTracker mockProgressTracker = mock(ProgressTracker.class); - when(mockJobProcessor.progressTracker(any())).thenReturn(mockProgressTracker); + when(mockJob.progressTracker()).thenReturn(Optional.ofNullable(mockProgressTracker)); when(mockJob.progress()).thenReturn(0f); when(mockJob.withProgress(anyFloat())).thenReturn(mockJob); @@ -679,14 +733,14 @@ public void test_Job_NotRetryable() throws Exception { // Verify the job was not retried verify(mockRetryStrategy, times(1)).shouldRetry(any(), any()); - verify(mockJobQueue, never()).putJobBackInQueue(any()); - verify(mockJobQueue, times(1)).removeJob(mockJob.id()); + verify(mockJobQueue, times(1)).putJobBackInQueue(any()); + verify(mockJobQueue, times(1)).removeJobFromQueue(mockJob.id()); // Capture and verify the error details - ArgumentCaptor errorDetailCaptor = ArgumentCaptor.forClass(ErrorDetail.class); - verify(mockJob).markAsFailed(errorDetailCaptor.capture()); - ErrorDetail capturedErrorDetail = errorDetailCaptor.getValue(); - assertEquals("Non-retryable error", capturedErrorDetail.exception().getMessage()); + ArgumentCaptor jobResultCaptor = ArgumentCaptor.forClass(JobResult.class); + verify(mockJob).markAsFailed(jobResultCaptor.capture()); + ErrorDetail capturedErrorDetail = jobResultCaptor.getValue().errorDetail().get(); + assertEquals("Non-retryable error", capturedErrorDetail.message()); // Stop the job queue jobQueueManagerAPI.close(); @@ -713,10 +767,15 @@ public void test_JobProgressTracking() throws Exception { jobState.set(inv.getArgument(0)); return mockJob; }); - when(mockJob.markAsCompleted()).thenAnswer(inv -> { + when(mockJob.markAsRunning()).thenAnswer(inv -> { + jobState.set(JobState.RUNNING); + return mockJob; + }); + when(mockJob.markAsCompleted(any())).thenAnswer(inv -> { jobState.set(JobState.COMPLETED); return mockJob; }); + when(mockJob.withProgressTracker(any(DefaultProgressTracker.class))).thenReturn(mockJob); when(mockJob.progress()).thenAnswer(inv -> jobProgress.get()); when(mockJob.withProgress(anyFloat())).thenAnswer(inv -> { @@ -730,7 +789,7 @@ public void test_JobProgressTracking() throws Exception { // Create a real ProgressTracker ProgressTracker realProgressTracker = new DefaultProgressTracker(); - when(mockJobProcessor.progressTracker(any())).thenReturn(realProgressTracker); + when(mockJob.progressTracker()).thenReturn(Optional.of(realProgressTracker)); // Make the circuit breaker always allow requests when(mockCircuitBreaker.allowRequest()).thenReturn(true); @@ -756,7 +815,7 @@ public void test_JobProgressTracking() throws Exception { } Job job = inv.getArgument(0); - job.markAsCompleted(); + job.markAsCompleted(any()); return null; }).when(mockJobProcessor).process(any()); @@ -765,6 +824,9 @@ public void test_JobProgressTracking() throws Exception { progressUpdates.add(job.progress()); }); + when(mockJobQueue.getUpdatedJobsSince(anySet(), any(LocalDateTime.class))) + .thenAnswer(invocation -> Collections.singletonList(mockJob)); + // Start the job queue jobQueueManagerAPI.start(); @@ -830,10 +892,11 @@ public void test_CircuitBreaker_Opens() throws Exception { when(mockJobQueue.getJob(anyString())).thenReturn(failingJob); when(failingJob.withState(any())).thenReturn(failingJob); when(failingJob.markAsFailed(any())).thenReturn(failingJob); + when(failingJob.markAsRunning()).thenReturn(failingJob); // Configure progress tracker ProgressTracker mockProgressTracker = mock(ProgressTracker.class); - when(mockJobProcessor.progressTracker(any())).thenReturn(mockProgressTracker); + when(failingJob.progressTracker()).thenReturn(Optional.ofNullable(mockProgressTracker)); when(failingJob.progress()).thenReturn(0f); when(failingJob.withProgress(anyFloat())).thenReturn(failingJob); @@ -844,9 +907,11 @@ public void test_CircuitBreaker_Opens() throws Exception { CircuitBreaker circuitBreaker = new CircuitBreaker(5, 60000); // Create JobQueueManagerAPIImpl with the real CircuitBreaker - JobQueueManagerAPI jobQueueManagerAPI = new JobQueueManagerAPIImpl( - mockJobQueue, new JobQueueConfig(1), circuitBreaker, mockRetryStrategy + JobQueueManagerAPI jobQueueManagerAPI = newJobQueueManagerAPI( + mockJobQueue, circuitBreaker, mockRetryStrategy, eventProducer, + 1, 1000 ); + jobQueueManagerAPI.registerProcessor("testQueue", mockJobProcessor); // Start the job queue @@ -894,11 +959,13 @@ public void test_CircuitBreaker_Closes() throws Exception { when(mockJobQueue.nextJob()).thenReturn(mockJob); when(mockJobQueue.getJob(anyString())).thenReturn(mockJob); when(mockJob.withState(any())).thenReturn(mockJob); + when(mockJob.markAsRunning()).thenReturn(mockJob); when(mockJob.markAsFailed(any())).thenReturn(mockJob); + when(mockJob.withProgressTracker(any(DefaultProgressTracker.class))).thenReturn(mockJob); // Configure progress tracker ProgressTracker mockProgressTracker = mock(ProgressTracker.class); - when(mockJobProcessor.progressTracker(any())).thenReturn(mockProgressTracker); + when(mockJob.progressTracker()).thenReturn(Optional.ofNullable(mockProgressTracker)); when(mockJob.progress()).thenReturn(0f); when(mockJob.withProgress(anyFloat())).thenReturn(mockJob); @@ -910,12 +977,12 @@ public void test_CircuitBreaker_Closes() throws Exception { } Job processingJob = inv.getArgument(0); - processingJob.markAsCompleted(); + processingJob.markAsCompleted(any()); return null; }).when(mockJobProcessor).process(any()); AtomicReference jobState = new AtomicReference<>(JobState.PENDING); - when(mockJob.markAsCompleted()).thenAnswer(inv -> { + when(mockJob.markAsCompleted(any())).thenAnswer(inv -> { jobState.set(JobState.COMPLETED); return mockJob; }); @@ -925,8 +992,9 @@ public void test_CircuitBreaker_Closes() throws Exception { 1000); // Short reset timeout for testing // Create JobQueueManagerAPIImpl with the real CircuitBreaker - JobQueueManagerAPI jobQueueManagerAPI = new JobQueueManagerAPIImpl( - mockJobQueue, new JobQueueConfig(1), circuitBreaker, mockRetryStrategy + JobQueueManagerAPI jobQueueManagerAPI = newJobQueueManagerAPI( + mockJobQueue, circuitBreaker, mockRetryStrategy, eventProducer, + 1, 1000 ); jobQueueManagerAPI.registerProcessor("testQueue", mockJobProcessor); @@ -970,10 +1038,13 @@ public void test_CircuitBreaker_Reset() throws Exception { when(mockJobQueue.getJob(anyString())).thenReturn(failingJob); when(failingJob.withState(any())).thenReturn(failingJob); when(failingJob.markAsFailed(any())).thenReturn(failingJob); + when(failingJob.markAsRunning()).thenReturn(failingJob); + when(failingJob.withProgressTracker(any(DefaultProgressTracker.class))).thenReturn( + failingJob); // Configure progress tracker ProgressTracker mockProgressTracker = mock(ProgressTracker.class); - when(mockJobProcessor.progressTracker(any())).thenReturn(mockProgressTracker); + when(failingJob.progressTracker()).thenReturn(Optional.ofNullable(mockProgressTracker)); when(failingJob.progress()).thenReturn(0f); when(failingJob.withProgress(anyFloat())).thenReturn(failingJob); @@ -984,8 +1055,9 @@ public void test_CircuitBreaker_Reset() throws Exception { CircuitBreaker circuitBreaker = new CircuitBreaker(5, 60000); // Create JobQueueManagerAPIImpl with the real CircuitBreaker - JobQueueManagerAPI jobQueueManagerAPI = new JobQueueManagerAPIImpl( - mockJobQueue, new JobQueueConfig(1), circuitBreaker, mockRetryStrategy + JobQueueManagerAPI jobQueueManagerAPI = newJobQueueManagerAPI( + mockJobQueue, circuitBreaker, mockRetryStrategy, eventProducer, + 1, 1000 ); jobQueueManagerAPI.registerProcessor("testQueue", mockJobProcessor); @@ -1014,34 +1086,57 @@ mockJobQueue, new JobQueueConfig(1), circuitBreaker, mockRetryStrategy /** * Method to test: cancelJob in JobQueueManagerAPI * Given Scenario: Valid job ID for a cancellable job is provided - * ExpectedResult: Job is successfully cancelled and its status is updated + * ExpectedResult: Job is successfully canceled and its status is updated */ @Test - public void test_simple_cancelJob() { + public void test_simple_cancelJob2() + throws DotDataException, JobQueueDataException, JobNotFoundException, JobCancellationException { + + class TestJobProcessor implements JobProcessor, Cancellable { + + @Override + public void process(Job job) throws JobProcessingException { + } + @Override + public void cancel(Job job) { + } + + @Override + public Map getResultMetadata(Job job) { + return null; + } + } + + // Create a mock job Job mockJob = mock(Job.class); when(mockJobQueue.getJob("job123")).thenReturn(mockJob); when(mockJob.queueName()).thenReturn("testQueue"); when(mockJob.id()).thenReturn("job123"); when(mockJob.withState(any())).thenReturn(mockJob); - when(mockJobProcessor.canCancel(mockJob)).thenReturn(true); + // Create a mock CancellableJobProcessor + TestJobProcessor mockCancellableProcessor = mock(TestJobProcessor.class); + + // Set up the job queue manager to return our mock cancellable processor + jobQueueManagerAPI.registerProcessor("testQueue", mockCancellableProcessor); + // Perform the cancellation jobQueueManagerAPI.cancelJob("job123"); - verify(mockJobProcessor).cancel(mockJob); - verify(mockJobQueue).updateJobStatus(any(Job.class)); + // Verify that the cancel method was called on our mock processor + verify(mockCancellableProcessor).cancel(mockJob); } /** * Method to test: Job cancellation in JobQueueManagerAPI - * Given Scenario: Running job is cancelled - * ExpectedResult: Job is successfully cancelled and its state transitions are correct + * Given Scenario: Running job is canceled + * ExpectedResult: Job is successfully canceled and its state transitions are correct */ @Test public void test_complex_cancelJob() throws Exception { - class TestJobProcessor implements JobProcessor { + class TestJobProcessor implements JobProcessor, Cancellable { private final AtomicBoolean cancellationRequested = new AtomicBoolean(false); private final CountDownLatch processingStarted = new CountDownLatch(1); @@ -1050,24 +1145,13 @@ class TestJobProcessor implements JobProcessor { @Override public void process(Job job) throws JobProcessingException { processingStarted.countDown(); - try { - - // Simulate work and wait for cancellation - Awaitility.await() - .pollInterval(100, TimeUnit.MILLISECONDS) - .atMost(30, TimeUnit.SECONDS) - .until(cancellationRequested::get); - - throw new InterruptedException("Job cancelled"); - } catch (InterruptedException e) { - processingCompleted.countDown(); - throw new JobProcessingException(job.id(), "Job was cancelled", e); - } - } + // Simulate work and wait for cancellation + Awaitility.await() + .pollInterval(100, TimeUnit.MILLISECONDS) + .atMost(30, TimeUnit.SECONDS) + .until(cancellationRequested::get); - @Override - public boolean canCancel(Job job) { - return true; + processingCompleted.countDown(); } @Override @@ -1075,6 +1159,11 @@ public void cancel(Job job) { cancellationRequested.set(true); } + @Override + public Map getResultMetadata(Job job) { + return null; + } + public boolean awaitProcessingStart(long timeout, TimeUnit unit) throws InterruptedException { return processingStarted.await(timeout, unit); @@ -1086,27 +1175,48 @@ public boolean awaitProcessingCompleted(long timeout, TimeUnit unit) } } - // Create a real job - Job job = Job.builder() - .id("job123") - .queueName("testQueue") - .state(JobState.PENDING) - .build(); + // Create a mock job + Job mockJob = mock(Job.class); + when(mockJob.id()).thenReturn("job123"); + when(mockJob.queueName()).thenReturn("testQueue"); // Use our TestJobProcessor TestJobProcessor testJobProcessor = new TestJobProcessor(); // Configure JobQueue - when(mockJobQueue.getJob("job123")).thenReturn(job); - when(mockJobQueue.nextJob()).thenReturn(job).thenReturn(null); + when(mockJobQueue.getJob("job123")).thenReturn(mockJob); + when(mockJobQueue.nextJob()).thenReturn(mockJob).thenReturn(null); + when(mockJobQueue.hasJobBeenInState(any(), eq(JobState.CANCELLING))).thenReturn(true); // List to capture job state updates List stateUpdates = new CopyOnWriteArrayList<>(); - // Set up a job watcher to capture state updates - jobQueueManagerAPI.watchJob("job123", updatedJob -> { - stateUpdates.add(updatedJob.state()); + when(mockJob.withState(any())).thenAnswer(inv -> { + stateUpdates.add(inv.getArgument(0)); + return mockJob; + }); + when(mockJob.markAsRunning()).thenAnswer(inv -> { + stateUpdates.add(JobState.RUNNING); + return mockJob; }); + when(mockJob.markAsCanceled(any())).thenAnswer(inv -> { + stateUpdates.add(JobState.CANCELED); + return mockJob; + }); + when(mockJob.markAsCompleted(any())).thenAnswer(inv -> { + stateUpdates.add(JobState.COMPLETED); + return mockJob; + }); + when(mockJob.markAsFailed(any())).thenAnswer(inv -> { + stateUpdates.add(JobState.FAILED); + return mockJob; + }); + when(mockJob.progress()).thenReturn(0f); + when(mockJob.withProgress(anyFloat())).thenReturn(mockJob); + when(mockJob.withProgressTracker(any(DefaultProgressTracker.class))).thenReturn(mockJob); + + when(mockJobQueue.getUpdatedJobsSince(anySet(), any(LocalDateTime.class))) + .thenAnswer(invocation -> Collections.singletonList(mockJob)); // Register the test processor jobQueueManagerAPI.registerProcessor("testQueue", testJobProcessor); @@ -1130,25 +1240,39 @@ public boolean awaitProcessingCompleted(long timeout, TimeUnit unit) .atMost(10, TimeUnit.SECONDS) .until(() -> testJobProcessor.awaitProcessingCompleted(100, TimeUnit.MILLISECONDS)); - // Wait for state updates to be captured Awaitility.await() - .atMost(5, TimeUnit.SECONDS) - .until(() -> stateUpdates.size() >= 3); - - // Verify the job state transitions - assertFalse("No state updates were captured", stateUpdates.isEmpty()); - assertEquals(JobState.PENDING, stateUpdates.get(0), "Initial state should be PENDING"); - assertTrue("Job state should have transitioned to RUNNING", - stateUpdates.contains(JobState.RUNNING)); - assertEquals(JobState.CANCELLED, stateUpdates.get(stateUpdates.size() - 1), - "Final state should be CANCELLED"); - - // Verify that the job status was updated in the queue - verify(mockJobQueue, timeout(5000)). - updateJobStatus(argThat(j -> j.state() == JobState.CANCELLED)); + .atMost(10, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .until(() -> stateUpdates.contains(JobState.CANCELED)); // Clean up jobQueueManagerAPI.close(); } + /** + * Creates a new instance of the JobQueueManagerAPI with the provided configurations. + * + * @param jobQueue The job queue to be managed. + * @param circuitBreaker The circuit breaker to handle job processing + * failures. + * @param retryStrategy The strategy to use for retrying failed jobs. + * @param threadPoolSize The size of the thread pool for job processing. + * @param pollJobUpdatesIntervalMilliseconds The interval in milliseconds for polling job + * updates. + * @return A newly created instance of JobQueueManagerAPI. + */ + private JobQueueManagerAPI newJobQueueManagerAPI(JobQueue jobQueue, + CircuitBreaker circuitBreaker, + RetryStrategy retryStrategy, + EventProducer eventProducer, + int threadPoolSize, int pollJobUpdatesIntervalMilliseconds) { + + final var realTimeJobMonitor = new RealTimeJobMonitor(); + + return new JobQueueManagerAPIImpl( + jobQueue, new JobQueueConfig(threadPoolSize, pollJobUpdatesIntervalMilliseconds), + circuitBreaker, retryStrategy, realTimeJobMonitor, eventProducer + ); + } + } diff --git a/dotcms-integration/src/test/java/com/dotcms/jobs/business/queue/PostgresJobQueueIntegrationTest.java b/dotcms-integration/src/test/java/com/dotcms/jobs/business/queue/PostgresJobQueueIntegrationTest.java new file mode 100644 index 000000000000..890354fd0285 --- /dev/null +++ b/dotcms-integration/src/test/java/com/dotcms/jobs/business/queue/PostgresJobQueueIntegrationTest.java @@ -0,0 +1,618 @@ +package com.dotcms.jobs.business.queue; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import com.dotcms.jobs.business.error.ErrorDetail; +import com.dotcms.jobs.business.job.Job; +import com.dotcms.jobs.business.job.JobPaginatedResult; +import com.dotcms.jobs.business.job.JobResult; +import com.dotcms.jobs.business.job.JobState; +import com.dotcms.jobs.business.queue.error.JobNotFoundException; +import com.dotcms.jobs.business.queue.error.JobQueueException; +import com.dotcms.util.IntegrationTestInitService; +import com.dotmarketing.common.db.DotConnect; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.util.Logger; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for PostgresJobQueue implementation + */ +public class PostgresJobQueueIntegrationTest { + + private static JobQueue jobQueue; + + @BeforeAll + static void setUp() throws Exception { + + //Setting web app environment + IntegrationTestInitService.getInstance().init(); + + jobQueue = new PostgresJobQueue(); + } + + @AfterEach + void cleanUpEach() { + clearJobs(); + } + + /** + * Method to test: createJob and getJob in PostgresJobQueue + * Given Scenario: A job is created with specific parameters + * ExpectedResult: The job can be retrieved and its properties match the input + */ + @Test + void test_createJob_and_getJob() throws JobQueueException { + + String queueName = "testQueue"; + Map parameters = new HashMap<>(); + parameters.put("key", "value"); + + String jobId = jobQueue.createJob(queueName, parameters); + assertNotNull(jobId); + + Job job = jobQueue.getJob(jobId); + assertNotNull(job); + assertEquals(queueName, job.queueName()); + assertEquals(JobState.PENDING, job.state()); + assertEquals(parameters, job.parameters()); + } + + /** + * Method to test: getActiveJobs in PostgresJobQueue + * Given Scenario: Multiple active jobs are created + * ExpectedResult: All active jobs are retrieved correctly + */ + @Test + void test_getActiveJobs() throws JobQueueException { + + String queueName = "testQueue"; + for (int i = 0; i < 5; i++) { + jobQueue.createJob(queueName, new HashMap<>()); + } + + JobPaginatedResult result = jobQueue.getActiveJobs(queueName, 1, 10); + assertEquals(5, result.jobs().size()); + assertEquals(5, result.total()); + } + + /** + * Method to test: getCompletedJobs in PostgresJobQueue + * Given Scenario: Multiple jobs are created and completed + * ExpectedResult: All completed jobs within the given time range are retrieved + */ + @Test + void testGetCompletedJobs() throws JobQueueException { + + String queueName = "testQueue"; + LocalDateTime startDate = LocalDateTime.now().minusDays(1); + LocalDateTime endDate = LocalDateTime.now().plusDays(1); + + // Create and complete some jobs + for (int i = 0; i < 3; i++) { + String jobId = jobQueue.createJob(queueName, new HashMap<>()); + Job job = jobQueue.getJob(jobId); + Job completedJob = job.markAsCompleted(null); + jobQueue.updateJobStatus(completedJob); + } + + JobPaginatedResult result = jobQueue.getCompletedJobs(queueName, startDate, endDate, 1, 10); + assertEquals(3, result.jobs().size()); + assertEquals(3, result.total()); + result.jobs().forEach(job -> assertEquals(JobState.COMPLETED, job.state())); + } + + /** + * Method to test: getFailedJobs in PostgresJobQueue + * Given Scenario: Multiple jobs are created and set to failed state + * ExpectedResult: All failed jobs are retrieved correctly + */ + @Test + void test_getFailedJobs() throws JobQueueException { + + // Create and fail some jobs + for (int i = 0; i < 2; i++) { + String jobId = jobQueue.createJob("testQueue", new HashMap<>()); + Job job = jobQueue.getJob(jobId); + Job failedJob = Job.builder().from(job) + .state(JobState.FAILED) + .build(); + jobQueue.updateJobStatus(failedJob); + } + + JobPaginatedResult result = jobQueue.getFailedJobs(1, 10); + assertEquals(2, result.jobs().size()); + assertEquals(2, result.total()); + result.jobs().forEach(job -> assertEquals(JobState.FAILED, job.state())); + } + + /** + * Method to test: updateJobStatus in PostgresJobQueue + * Given Scenario: A job's status is updated + * ExpectedResult: The job's status is correctly reflected in the database + */ + @Test + void test_updateJobStatus() throws JobQueueException { + + String jobId = jobQueue.createJob("testQueue", new HashMap<>()); + Job job = jobQueue.getJob(jobId); + + Job updatedJob = Job.builder().from(job) + .state(JobState.RUNNING) + .progress(0.5f) + .build(); + + jobQueue.updateJobStatus(updatedJob); + + Job fetchedJob = jobQueue.getJob(jobId); + assertEquals(JobState.RUNNING, fetchedJob.state()); + assertEquals(0.5f, fetchedJob.progress(), 0.001); + } + + /** + * Method to test: nextJob in PostgresJobQueue + * Given Scenario: Multiple threads attempt to get the next job concurrently + * ExpectedResult: Each job is processed exactly once and all jobs are eventually completed + */ + @Test + void test_nextJob() throws Exception { + + final int NUM_JOBS = 10; + final int NUM_THREADS = 5; + String queueName = "testQueue"; + + // Create jobs + Set createdJobIds = new HashSet<>(); + for (int i = 0; i < NUM_JOBS; i++) { + String jobId = jobQueue.createJob(queueName, new HashMap<>()); + createdJobIds.add(jobId); + } + + // Set to keep track of processed job IDs + Set processedJobIds = Collections.synchronizedSet(new HashSet<>()); + + // Create and start threads + List threads = new ArrayList<>(); + for (int i = 0; i < NUM_THREADS; i++) { + Thread thread = new Thread(() -> { + try { + while (true) { + Job nextJob = jobQueue.nextJob(); + if (nextJob == null) { + break; // No more jobs to process + } + // Ensure this job hasn't been processed before + assertTrue(processedJobIds.add(nextJob.id()), + "Job " + nextJob.id() + " was processed more than once"); + assertEquals(JobState.PENDING, nextJob.state()); + + // Simulate some processing time + Awaitility.await().atMost(5, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS).until(() -> { + return true; + }); + + // Mark job as completed + Job completedJob = nextJob.markAsCompleted(null); + jobQueue.updateJobStatus(completedJob); + } + } catch (Exception e) { + fail("Exception in thread: " + e.getMessage()); + } + }); + threads.add(thread); + thread.start(); + } + + // Wait for all threads to complete + for (Thread thread : threads) { + thread.join(); + } + + // Verify all jobs were processed + assertEquals(NUM_JOBS, processedJobIds.size(), "Not all jobs were processed"); + assertEquals(createdJobIds, processedJobIds, "Processed jobs don't match created jobs"); + + // Verify no more jobs are available + assertNull(jobQueue.nextJob(), "There should be no more jobs available"); + + // Verify all jobs are in COMPLETED state + for (String jobId : createdJobIds) { + Job job = jobQueue.getJob(jobId); + assertEquals(JobState.COMPLETED, job.state(), + "Job " + jobId + " is not in COMPLETED state"); + } + } + + /** + * Method to test: getJob in PostgresJobQueue with non-existent ID + * Given Scenario: Attempt to retrieve a job with a non-existent ID + * ExpectedResult: JobNotFoundException is thrown + */ + @Test + void testJobNotFound() { + assertThrows(JobNotFoundException.class, () -> jobQueue.getJob("non-existent-id")); + } + + /** + * Method to test: updateJobProgress in PostgresJobQueue + * Given Scenario: A job's progress is updated multiple times + * ExpectedResult: The job's progress is correctly updated in the database + */ + @Test + void test_updateJobProgress() throws JobQueueException { + + String jobId = jobQueue.createJob("testQueue", new HashMap<>()); + + jobQueue.updateJobProgress(jobId, 0.75f); + + Job updatedJob = jobQueue.getJob(jobId); + assertEquals(0.75f, updatedJob.progress(), 0.001); + + jobQueue.updateJobProgress(jobId, 0.85f); + + updatedJob = jobQueue.getJob(jobId); + assertEquals(0.85f, updatedJob.progress(), 0.001); + } + + /** + * Method to test: getJobs in PostgresJobQueue + * Given Scenario: Jobs with various states are created + * ExpectedResult: All jobs are retrieved correctly with proper pagination + */ + @Test + void test_getJobs() throws JobQueueException { + + // Create a mix of jobs with different states + String queueName = "testQueue"; + for (int i = 0; i < 3; i++) { + jobQueue.createJob(queueName, new HashMap<>()); + } + String runningJobId = jobQueue.createJob(queueName, new HashMap<>()); + Job runningJob = jobQueue.getJob(runningJobId); + jobQueue.updateJobStatus(runningJob.withState(JobState.RUNNING)); + + String completedJobId = jobQueue.createJob(queueName, new HashMap<>()); + Job completedJob = jobQueue.getJob(completedJobId); + jobQueue.updateJobStatus(completedJob.markAsCompleted(null)); + + // Get all jobs + JobPaginatedResult result = jobQueue.getJobs(1, 10); + + assertEquals(5, result.jobs().size()); + assertEquals(5, result.total()); + assertEquals(1, result.page()); + assertEquals(10, result.pageSize()); + + // Verify job states + Map stateCounts = new HashMap<>(); + for (Job job : result.jobs()) { + stateCounts.put(job.state(), stateCounts.getOrDefault(job.state(), 0) + 1); + } + assertEquals(3, stateCounts.getOrDefault(JobState.PENDING, 0)); + assertEquals(1, stateCounts.getOrDefault(JobState.RUNNING, 0)); + assertEquals(1, stateCounts.getOrDefault(JobState.COMPLETED, 0)); + } + + /** + * Method to test: getUpdatedJobsSince in PostgresJobQueue + * Given Scenario: Jobs are created and updated at different times + * ExpectedResult: Only jobs updated after the specified time are retrieved + */ + @Test + void test_getUpdatedJobsSince() throws JobQueueException, InterruptedException { + + String queueName = "testQueue"; + + // Create initial jobs + String job1Id = jobQueue.createJob(queueName, new HashMap<>()); + String job2Id = jobQueue.createJob(queueName, new HashMap<>()); + + Awaitility.await().atMost(1, TimeUnit.SECONDS) + .pollInterval(50, TimeUnit.MILLISECONDS) + .until(() -> true);// Ensure some time passes + LocalDateTime checkpointTime = LocalDateTime.now(); + Awaitility.await().atMost(1, TimeUnit.SECONDS) + .pollInterval(50, TimeUnit.MILLISECONDS) + .until(() -> true);// Ensure some more time passes + + // Update job1 and create a new job after the checkpoint + Job job1 = jobQueue.getJob(job1Id); + jobQueue.updateJobStatus(job1.withState(JobState.RUNNING)); + String job3Id = jobQueue.createJob(queueName, new HashMap<>()); + + Set jobIdsToCheck = new HashSet<>(Arrays.asList(job1Id, job2Id, job3Id)); + List updatedJobs = jobQueue.getUpdatedJobsSince(jobIdsToCheck, checkpointTime); + + assertEquals(2, updatedJobs.size()); + Set updatedJobIds = updatedJobs.stream().map(Job::id).collect(Collectors.toSet()); + assertTrue(updatedJobIds.contains(job1Id)); + assertTrue(updatedJobIds.contains(job3Id)); + assertFalse(updatedJobIds.contains(job2Id)); + } + + /** + * Method to test: putJobBackInQueue in PostgresJobQueue + * Given Scenario: A failed job is put back into the queue + * ExpectedResult: The job is reset to PENDING state and can be retrieved by nextJob + */ + @Test + void test_putJobBackInQueue() throws JobQueueException { + + String queueName = "testQueue"; + String jobId = jobQueue.createJob(queueName, new HashMap<>()); + + // Simulate job processing + Job job = jobQueue.getJob(jobId); + jobQueue.updateJobStatus(job.withState(JobState.RUNNING)); + + // Simulate job failure + final var jobResult = JobResult.builder() + .errorDetail(ErrorDetail.builder() + .message("Simulated error") + .stackTrace(stackTrace(new RuntimeException("Simulated error"))) + .exceptionClass("java.lang.RuntimeException") + .timestamp(LocalDateTime.now()) + .processingStage("Simulated stage") + .build()) + .build(); + jobQueue.updateJobStatus(job.markAsFailed(jobResult)); + + // Put the job back in the queue + jobQueue.putJobBackInQueue(job); + + // Verify the job maintains the FAILED state, needed for the retry mechanism to work + Job retrievedJob = jobQueue.getJob(jobId); + assertEquals(JobState.FAILED, retrievedJob.state()); + + // Verify the job can be retrieved by nextJob + Job nextJob = jobQueue.nextJob(); + assertNotNull(nextJob); + assertEquals(jobId, nextJob.id()); + assertEquals(JobState.FAILED, nextJob.state()); + } + + /** + * Method to test: removeJobFromQueue in PostgresJobQueue + * Given Scenario: A job is removed from the queue + * ExpectedResult: The job cannot be retrieved by nextJob after removal + */ + @Test + void test_removeJobFromQueue() throws JobQueueException { + + String queueName = "testQueue"; + String jobId = jobQueue.createJob(queueName, new HashMap<>()); + + // Verify job exists + Job job = jobQueue.getJob(jobId); + assertNotNull(job); + + // Remove the job + jobQueue.removeJobFromQueue(jobId); + + // Verify job is not returned by nextJob + assertNull(jobQueue.nextJob()); + } + + /** + * Method to test: createJob, updateJobStatus, and getJob in PostgresJobQueue + * Given Scenario: A job is created, all its fields are modified, the job is updated, + * and then retrieved again + * ExpectedResult: All job fields are correctly updated and retrieved, demonstrating + * proper persistence and retrieval of all job attributes + */ + @Test + void test_createUpdateAndRetrieveJob() throws JobQueueException { + + String queueName = "testQueue"; + Map initialParameters = new HashMap<>(); + initialParameters.put("initialKey", "initialValue"); + + // Create initial job + String jobId = jobQueue.createJob(queueName, initialParameters); + Job initialJob = jobQueue.getJob(jobId); + assertNotNull(initialJob); + + // Modify all fields + JobResult jobResult = JobResult.builder() + .errorDetail(ErrorDetail.builder() + .message("Test error") + .exceptionClass("TestException") + .timestamp(LocalDateTime.now()) + .stackTrace("Test stack trace") + .processingStage("Test stage") + .build()) + .metadata(Collections.singletonMap("metaKey", "metaValue")) + .build(); + + Job updatedJob = Job.builder() + .from(initialJob) + .state(JobState.COMPLETED) + .progress(0.75f) + .startedAt(Optional.of(LocalDateTime.now().minusHours(1))) + .completedAt(Optional.of(LocalDateTime.now())) + .retryCount(2) + .result(Optional.of(jobResult)) + .build(); + + // Update the job + jobQueue.updateJobStatus(updatedJob); + + // Retrieve the updated job + Job retrievedJob = jobQueue.getJob(jobId); + + // Verify all fields + assertEquals(jobId, retrievedJob.id()); + assertEquals(queueName, retrievedJob.queueName()); + assertEquals(JobState.COMPLETED, retrievedJob.state()); + assertEquals(initialParameters, retrievedJob.parameters()); + assertEquals(0.75f, retrievedJob.progress(), 0.001); + assertTrue(retrievedJob.startedAt().isPresent()); + assertTrue(retrievedJob.completedAt().isPresent()); + assertNotNull(retrievedJob.executionNode()); + assertEquals(2, retrievedJob.retryCount()); + assertTrue(retrievedJob.result().isPresent()); + assertEquals("Test error", + retrievedJob.result().get().errorDetail().get().message()); + assertEquals("Test stack trace", + retrievedJob.result().get().errorDetail().get().stackTrace()); + assertEquals("TestException", + retrievedJob.result().get().errorDetail().get().exceptionClass()); + assertEquals("Test stage", + retrievedJob.result().get().errorDetail().get().processingStage()); + assertEquals(Collections.singletonMap("metaKey", "metaValue"), + retrievedJob.result().get().metadata().get()); + } + + /** + * Method to test: getJobs in PostgresJobQueue with pagination + * Given Scenario: Multiple jobs are created and retrieved using pagination + * ExpectedResult: Jobs are correctly paginated and retrieved in the expected order + */ + @Test + void test_getJobsPagination() throws JobQueueException { + + String queueName = "paginationTestQueue"; + int totalJobs = 25; + int pageSize = 10; + + // Create jobs + List createdJobIds = new ArrayList<>(); + for (int i = 0; i < totalJobs; i++) { + Map params = new HashMap<>(); + params.put("index", i); + String jobId = jobQueue.createJob(queueName, params); + createdJobIds.add(jobId); + } + + // Test first page + JobPaginatedResult page1 = jobQueue.getJobs(1, pageSize); + assertEquals(pageSize, page1.jobs().size()); + assertEquals(totalJobs, page1.total()); + assertEquals(1, page1.page()); + assertEquals(pageSize, page1.pageSize()); + + // Test second page + JobPaginatedResult page2 = jobQueue.getJobs(2, pageSize); + assertEquals(pageSize, page2.jobs().size()); + assertEquals(totalJobs, page2.total()); + assertEquals(2, page2.page()); + assertEquals(pageSize, page2.pageSize()); + + // Test last page + JobPaginatedResult page3 = jobQueue.getJobs(3, pageSize); + assertEquals(5, page3.jobs().size()); // 25 total, 20 in first two pages, 5 in last + assertEquals(totalJobs, page3.total()); + assertEquals(3, page3.page()); + assertEquals(pageSize, page3.pageSize()); + + // Verify no overlap between pages + Set jobIdsPage1 = page1.jobs().stream().map(Job::id).collect(Collectors.toSet()); + Set jobIdsPage2 = page2.jobs().stream().map(Job::id).collect(Collectors.toSet()); + Set jobIdsPage3 = page3.jobs().stream().map(Job::id).collect(Collectors.toSet()); + + assertEquals(pageSize, jobIdsPage1.size()); + assertEquals(pageSize, jobIdsPage2.size()); + assertEquals(5, jobIdsPage3.size()); + + assertTrue(Collections.disjoint(jobIdsPage1, jobIdsPage2)); + assertTrue(Collections.disjoint(jobIdsPage1, jobIdsPage3)); + assertTrue(Collections.disjoint(jobIdsPage2, jobIdsPage3)); + + // Verify all jobs are retrieved + Set allRetrievedJobIds = new HashSet<>(); + allRetrievedJobIds.addAll(jobIdsPage1); + allRetrievedJobIds.addAll(jobIdsPage2); + allRetrievedJobIds.addAll(jobIdsPage3); + assertEquals(new HashSet<>(createdJobIds), allRetrievedJobIds); + + // Test invalid page + JobPaginatedResult invalidPage = jobQueue.getJobs(10, pageSize); + assertTrue(invalidPage.jobs().isEmpty()); + assertEquals(totalJobs, invalidPage.total()); + assertEquals(10, invalidPage.page()); + assertEquals(pageSize, invalidPage.pageSize()); + } + + /** + * Method to test: hasJobBeenInState in PostgresJobQueue + * Given Scenario: A job is created and its state is updated + * ExpectedResult: The job's state history is correctly validated + */ + @Test + void test_hasJobBeenInState() throws JobQueueException { + + String queueName = "testQueue"; + String jobId = jobQueue.createJob(queueName, new HashMap<>()); + + Job job = jobQueue.getJob(jobId); + + // Make sure it is validating properly the given states + + jobQueue.updateJobStatus(job.withState(JobState.RUNNING)); + assertTrue(jobQueue.hasJobBeenInState(jobId, JobState.RUNNING)); + + assertFalse(jobQueue.hasJobBeenInState(jobId, JobState.CANCELED)); + + jobQueue.updateJobStatus(job.withState(JobState.COMPLETED)); + assertTrue(jobQueue.hasJobBeenInState(jobId, JobState.COMPLETED)); + + assertFalse(jobQueue.hasJobBeenInState(jobId, JobState.CANCELLING)); + + jobQueue.updateJobStatus(job.withState(JobState.CANCELLING)); + assertTrue(jobQueue.hasJobBeenInState(jobId, JobState.CANCELLING)); + } + + /** + * Helper method to clear all jobs from the database + */ + private void clearJobs() { + try { + new DotConnect().setSQL("delete from job_history").loadResult(); + new DotConnect().setSQL("delete from job_queue").loadResult(); + new DotConnect().setSQL("delete from job").loadResult(); + } catch (DotDataException e) { + Logger.error(this, "Error clearing jobs", e); + } + } + + /** + * Generates and returns the stack trace of the exception as a string. This is a derived value + * and will be computed only when accessed. + * + * @param exception The exception for which to generate the stack trace. + * @return A string representation of the exception's stack trace, or null if no exception is + * present. + */ + private String stackTrace(final Throwable exception) { + if (exception != null) { + return Arrays.stream(exception.getStackTrace()) + .map(StackTraceElement::toString) + .reduce((a, b) -> a + "\n" + b) + .orElse(""); + } + return null; + } + +} \ No newline at end of file diff --git a/dotcms-integration/src/test/java/com/dotcms/util/IntegrationTestInitService.java b/dotcms-integration/src/test/java/com/dotcms/util/IntegrationTestInitService.java index 2ea04111cde2..e2d589cb1ae5 100644 --- a/dotcms-integration/src/test/java/com/dotcms/util/IntegrationTestInitService.java +++ b/dotcms-integration/src/test/java/com/dotcms/util/IntegrationTestInitService.java @@ -2,6 +2,16 @@ import com.dotcms.business.bytebuddy.ByteBuddyFactory; import com.dotcms.config.DotInitializationService; +import com.dotcms.jobs.business.api.JobQueueConfig; +import com.dotcms.jobs.business.api.JobQueueConfigProducer; +import com.dotcms.jobs.business.api.JobQueueManagerAPIImpl; +import com.dotcms.jobs.business.api.events.EventProducer; +import com.dotcms.jobs.business.api.events.RealTimeJobMonitor; +import com.dotcms.jobs.business.error.CircuitBreaker; +import com.dotcms.jobs.business.error.RetryStrategy; +import com.dotcms.jobs.business.error.RetryStrategyProducer; +import com.dotcms.jobs.business.queue.JobQueue; +import com.dotcms.jobs.business.queue.JobQueueProducer; import com.dotcms.repackage.org.apache.struts.Globals; import com.dotcms.repackage.org.apache.struts.config.ModuleConfig; import com.dotcms.repackage.org.apache.struts.config.ModuleConfigFactory; @@ -12,16 +22,15 @@ import com.dotmarketing.util.Config; import com.dotmarketing.util.Logger; import com.liferay.util.SystemProperties; +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.awaitility.Awaitility; import org.jboss.weld.bootstrap.api.helpers.RegistrySingletonProvider; import org.jboss.weld.environment.se.Weld; import org.jboss.weld.environment.se.WeldContainer; import org.mockito.Mockito; -import java.time.Duration; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - /** * Sets up the web environment needed to execute integration tests without a server application * Created by nollymar on 9/29/16. @@ -51,6 +60,17 @@ public void init() throws Exception { if (initCompleted.compareAndSet(false, true)) { weld = new Weld().containerId(RegistrySingletonProvider.STATIC_INSTANCE) + .beanClasses( + JobQueueManagerAPIImpl.class, + JobQueueConfig.class, + JobQueue.class, + RetryStrategy.class, + CircuitBreaker.class, + JobQueueProducer.class, + JobQueueConfigProducer.class, + RetryStrategyProducer.class, + RealTimeJobMonitor.class, + EventProducer.class) .initialize(); System.setProperty(TestUtil.DOTCMS_INTEGRATION_TEST, TestUtil.DOTCMS_INTEGRATION_TEST);