Persist terminal "aborted" state so dead jobs stop counting as queued

A job with `completed IS NULL` is treated as queued/in-progress by
isQueued(), the admin "in_progress" filter, and the status badge. That
conflates three different things: pending, actively running, and
permanently failed. Once a job exhausts its retries it will never run
again, yet it keeps `completed IS NULL` forever — so:

* isQueued() keeps returning true, which wedges any non-concurrent
  caller (e.g. a QueueScheduler row) permanently behind the dead job;
* the admin shows it as "Running" instead of failed.

getFailedStatus() already distinguishes a retryable failure
("requeued") from a terminal one ("aborted"); this only persists that
verdict so the rest of the system can see it.

* Add `QueuedJobsTable::STATUS_ABORTED` and `markJobAborted()`.
* Processor stamps it when getFailedStatus() === aborted (alongside the
  existing maxAttemptsExhausted event).
* isQueued() and the "in_progress" search filter exclude aborted rows
  (null/other statuses are unaffected — fully backward compatible).
* The status badge renders aborted as "Failed", checked before the
  `fetched` branch so a job whose worker died on its last attempt no
  longer shows a stuck "Running".

Retryable failures are untouched: while attempts remain the job is
genuinely in flight and still counts as queued, so there is no early
re-dispatch or pile-up. flushFailedJobs() continues to prune old rows.

Author: dereuromark

src/Model/Filter/QueuedJobsCollection.php

@@ -6,6 +6,7 @@
 use Cake\Http\Exception\NotImplementedException;
 use Cake\I18n\DateTime;
 use Cake\ORM\Query\SelectQuery;
+use Queue\Model\Table\QueuedJobsTable;
 use Search\Model\Filter\FilterCollection;
 
 class QueuedJobsCollection extends FilterCollection {
@@ -32,9 +33,17 @@ public function initialize(): void {
 					if ($status === 'in_progress') {
 						$query->where([
 							'completed IS' => null,
+							// Exclude terminally-failed (aborted) jobs: they are
+							// done, not in progress, even without a completed stamp.
 							'OR' => [
-								'notbefore <=' => new DateTime(),
-								'notbefore IS' => null,
+								'status IS' => null,
+								'status !=' => QueuedJobsTable::STATUS_ABORTED,
+							],
+							'AND' => [
+								'OR' => [
+									'notbefore <=' => new DateTime(),
+									'notbefore IS' => null,
+								],
 							],
 						]);
 

src/Model/Table/QueuedJobsTable.php

@@ -76,6 +76,16 @@ class QueuedJobsTable extends Table {
 	 */
 	public const DAY = 86400;
 
+	/**
+	 * Terminal `status` value stamped on a job that has exhausted all of its
+	 * retries. Distinguishes a permanently-failed job (which will never run
+	 * again) from one that is pending, in progress, or still being retried —
+	 * all of which otherwise share `completed IS NULL`.
+	 *
+	 * @var string
+	 */
+	public const STATUS_ABORTED = 'aborted';
+
 	/**
 	 * @var array<string, string>
 	 */
@@ -291,6 +301,15 @@ public function isQueued(string $reference, ?string $jobTask = null): bool {
 		$conditions = [
 			'reference' => $reference,
 			'completed IS' => null,
+			// A job that exhausted its retries (status = aborted) will never
+			// run again, so it is not "queued" anymore even though it has no
+			// completed timestamp. Excluding it stops callers that gate on
+			// isQueued() — e.g. a non-concurrent scheduler row — from wedging
+			// permanently behind a dead job. The OR keeps null/other statuses.
+			'OR' => [
+				'status IS' => null,
+				'status !=' => static::STATUS_ABORTED,
+			],
 		];
 		if ($jobTask) {
 			$conditions['job_task'] = $jobTask;
@@ -769,6 +788,26 @@ public function markJobFailed(QueuedJob $job, ?string $failureMessage = null, ?s
 		return (bool)$this->save($job);
 	}
 
+	/**
+	 * Mark a job as terminally failed (aborted) after it has exhausted all
+	 * retries. Records the terminal `status` so the job is no longer reported
+	 * as queued/in-progress: `isQueued()` excludes it, the admin status filter
+	 * skips it, and the status badge renders it as failed rather than running.
+	 *
+	 * Call this only once the worker has determined no further attempts will
+	 * be made (see Processor, getFailedStatus() === self::STATUS_ABORTED).
+	 *
+	 * @param \Queue\Model\Entity\QueuedJob $job Job
+	 *
+	 * @return bool Success
+	 */
+	public function markJobAborted(QueuedJob $job): bool {
+		return (bool)$this->updateAll(
+			['status' => static::STATUS_ABORTED],
+			['id' => $job->id],
+		);
+	}
+
 	/**
 	 * Removes all failed jobs.
 	 *
@@ -1202,7 +1241,7 @@ public function getFailedStatus(QueuedJob $queuedTask, array $taskConfiguration)
 			return $failureMessageRequeued;
 		}
 
-		return 'aborted';
+		return static::STATUS_ABORTED;
 	}
 
 	/**

src/Queue/Processor.php

@@ -349,7 +349,13 @@ protected function runJob(QueuedJob $queuedJob, string $pid): void {
 			EventManager::instance()->dispatch($event);
 
 			// Dispatch event when job has exhausted all retries
-			if ($failedStatus === 'aborted') {
+			if ($failedStatus === $this->QueuedJobs::STATUS_ABORTED) {
+				// Persist the terminal state so the job stops counting as
+				// queued/in-progress (it will never run again). Without this
+				// it keeps `completed IS NULL` forever and wedges callers that
+				// gate on isQueued(), e.g. a non-concurrent scheduler row.
+				$this->QueuedJobs->markJobAborted($queuedJob);
+
 				$event = new Event('Queue.Job.maxAttemptsExhausted', $this, [
 					'job' => $queuedJob,
 					'failureMessage' => $failureMessage,

templates/element/Queue/status_badge.php

@@ -1,18 +1,23 @@
 <?php
+
 /**
  * Job Status Badge Element
  *
  * @var \Cake\View\View $this
  * @var \Queue\Model\Entity\QueuedJob $job The queued job entity
  */
+use Queue\Model\Table\QueuedJobsTable;
 
 $status = 'pending';
 $icon = 'clock';
 
 if ($job->completed) {
 	$status = 'completed';
 	$icon = 'check';
-} elseif ($job->failure_message) {
+} elseif ($job->status === QueuedJobsTable::STATUS_ABORTED || $job->failure_message) {
+	// Terminal: retries exhausted (status = aborted) or a recorded failure.
+	// Checked before `fetched` so a job whose worker died on its last attempt
+	// shows as Failed, not stuck "Running".
 	$status = 'failed';
 	$icon = 'times';
 } elseif ($job->fetched) {

tests/TestCase/Model/Table/QueuedJobsTableTest.php

@@ -834,6 +834,46 @@ public function testIsQueued() {
 		$this->assertFalse($result);
 	}
 
+	/**
+	 * A job that has exhausted its retries (status = aborted) is terminal and
+	 * must no longer count as queued, even though it has no completed stamp.
+	 * Normal null-status rows still count.
+	 *
+	 * @return void
+	 */
+	public function testIsQueuedExcludesAborted() {
+		$queuedJob = $this->QueuedJobs->newEntity([
+			'key' => 'key',
+			'job_task' => 'FooBar',
+			'reference' => 'foo-bar',
+		]);
+		$this->QueuedJobs->saveOrFail($queuedJob);
+		$this->assertTrue($this->QueuedJobs->isQueued('foo-bar'));
+
+		$this->QueuedJobs->markJobAborted($queuedJob);
+		$this->assertFalse($this->QueuedJobs->isQueued('foo-bar'));
+	}
+
+	/**
+	 * markJobAborted() stamps the terminal status without touching completed.
+	 *
+	 * @return void
+	 */
+	public function testMarkJobAborted() {
+		$queuedJob = $this->QueuedJobs->newEntity([
+			'key' => 'key',
+			'job_task' => 'FooBar',
+			'reference' => 'foo-bar',
+		]);
+		$this->QueuedJobs->saveOrFail($queuedJob);
+
+		$this->assertTrue($this->QueuedJobs->markJobAborted($queuedJob));
+
+		$reloaded = $this->QueuedJobs->get($queuedJob->id);
+		$this->assertSame(QueuedJobsTable::STATUS_ABORTED, $reloaded->status);
+		$this->assertNull($reloaded->completed);
+	}
+
 	/**
 	 * @return void
 	 */

tests/TestCase/Queue/ProcessorTest.php

@@ -271,6 +271,11 @@ public function testMaxAttemptsExhaustedEvent() {
 		// Verify event data
 		// The event was fired successfully (assertEventFired passed)
 		// We don't need to check the event data again since assertEventFired confirms it was fired
+
+		// The exhausted job must be stamped terminal so it no longer counts as
+		// queued/in-progress (otherwise isQueued() wedges callers behind it).
+		$reloaded = $QueuedJobs->get($job->id);
+		$this->assertSame(QueuedJobsTable::STATUS_ABORTED, $reloaded->status);
 	}
 
 	/**