Feat/clean crashed jobs

core/BackgroundJobs/CleanupBackgroundJobsJob.php

@@ -0,0 +1,59 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OC\Core\BackgroundJobs;
+
+use DateTimeImmutable;
+use OC\BackgroundJob\JobRuns;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\IJob;
+use OCP\BackgroundJob\JobStatus;
+use OCP\BackgroundJob\TimedJob;
+use OCP\Util;
+use Override;
+
+class CleanupBackgroundJobsJob extends TimedJob {
+	public function __construct(
+		ITimeFactory $time,
+		private readonly JobRuns $jobRuns,
+	) {
+		parent::__construct($time);
+		$this->setInterval(60 * 60);
+		$this->setTimeSensitivity(IJob::TIME_INSENSITIVE);
+	}
+
+	#[Override]
+	protected function run($argument): void {
+		$this->reapCrashedJobs();
+
+		// TODO Clean oldest jobs
+	}
+
+	private function reapCrashedJobs(): void {
+		$currentServerId = Util::getServerId();
+
+		foreach ($this->jobRuns->runningJobs(1000) as $job) {
+			if ($job->serverId !== $currentServerId) {
+				continue;
+			}
+			exec('ps -p ' . escapeshellarg((string)$job->pid) . ' -o cmd', $output, $result);
+			if (count($output) === 1 && $output[0] === 'CMD' && $result === 1) {
+				// Process doesn't exists anymore
+				$maxDuration = (new DateTimeImmutable())->diff($job->startedAt);
+				$maxDuration
+					= ($maxDuration->format('%a') * 24 * 60 * 60 * 1000)
+					+ ($maxDuration->format('%h') * 60 * 60 * 1000)
+					+ ($maxDuration->format('%m') * 60 * 1000)
+					+ ($maxDuration->format('%s') * 1000)
+					+ (int)($maxDuration->format('%f') / 1000);
+				$this->jobRuns->finished($job->runId, $maxDuration, 0, JobStatus::CRASHED);
+			}
+		}
+	}
+}

core/Command/Background/JobsHistory.php

@@ -75,7 +75,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 	private function formatLine(iterable $jobs): \Generator {
 		$jobsInfo = [];
 		$now = time();
-		$currentServerId = $this->config->getSystemValueInt('serverid', -1);
+		$currentServerId = Util::getServerId();
 		foreach ($jobs as $job) {
 			$status = match ($job->status) {
 				JobStatus::RUNNING => 'Running',

core/Command/Background/RunningJobs.php

@@ -12,6 +12,7 @@
 use OC\BackgroundJob\JobRuns;
 use OC\Core\Command\Base;
 use OCP\IConfig;
+use OCP\Util;
 use Override;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
@@ -60,7 +61,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
 	private function formatLine(iterable $jobs): \Generator {
 		$now = time();
-		$currentServerId = $this->config->getSystemValueInt('serverid', -1);
+		$currentServerId = Util::getServerId();
 		foreach ($jobs as $job) {
 			yield [
 				'Run ID' => $job->runId,

lib/composer/composer/autoload_classmap.php

@@ -1310,6 +1310,7 @@
     'OC\\Core\\AppInfo\\ConfigLexicon' => $baseDir . '/core/AppInfo/ConfigLexicon.php',
     'OC\\Core\\BackgroundJobs\\BackgroundCleanupUpdaterBackupsJob' => $baseDir . '/core/BackgroundJobs/BackgroundCleanupUpdaterBackupsJob.php',
     'OC\\Core\\BackgroundJobs\\CheckForUserCertificates' => $baseDir . '/core/BackgroundJobs/CheckForUserCertificates.php',
+    'OC\\Core\\BackgroundJobs\\CleanupBackgroundJobsJob' => $baseDir . '/core/BackgroundJobs/CleanupBackgroundJobsJob.php',
     'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => $baseDir . '/core/BackgroundJobs/CleanupLoginFlowV2.php',
     'OC\\Core\\BackgroundJobs\\ExpirePreviewsJob' => $baseDir . '/core/BackgroundJobs/ExpirePreviewsJob.php',
     'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => $baseDir . '/core/BackgroundJobs/GenerateMetadataJob.php',
@@ -2045,6 +2046,7 @@
     'OC\\Repair' => $baseDir . '/lib/private/Repair.php',
     'OC\\RepairException' => $baseDir . '/lib/private/RepairException.php',
     'OC\\Repair\\AddBruteForceCleanupJob' => $baseDir . '/lib/private/Repair/AddBruteForceCleanupJob.php',
+    'OC\\Repair\\AddCleanupBackgroundJobsJob' => $baseDir . '/lib/private/Repair/AddCleanupBackgroundJobsJob.php',
     'OC\\Repair\\AddCleanupDeletedUsersBackgroundJob' => $baseDir . '/lib/private/Repair/AddCleanupDeletedUsersBackgroundJob.php',
     'OC\\Repair\\AddCleanupUpdaterBackupsJob' => $baseDir . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php',
     'OC\\Repair\\AddMetadataGenerationJob' => $baseDir . '/lib/private/Repair/AddMetadataGenerationJob.php',

lib/composer/composer/autoload_static.php

@@ -1351,6 +1351,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Core\\AppInfo\\ConfigLexicon' => __DIR__ . '/../../..' . '/core/AppInfo/ConfigLexicon.php',
         'OC\\Core\\BackgroundJobs\\BackgroundCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/BackgroundCleanupUpdaterBackupsJob.php',
         'OC\\Core\\BackgroundJobs\\CheckForUserCertificates' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CheckForUserCertificates.php',
+        'OC\\Core\\BackgroundJobs\\CleanupBackgroundJobsJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CleanupBackgroundJobsJob.php',
         'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CleanupLoginFlowV2.php',
         'OC\\Core\\BackgroundJobs\\ExpirePreviewsJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/ExpirePreviewsJob.php',
         'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/GenerateMetadataJob.php',
@@ -2086,6 +2087,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Repair' => __DIR__ . '/../../..' . '/lib/private/Repair.php',
         'OC\\RepairException' => __DIR__ . '/../../..' . '/lib/private/RepairException.php',
         'OC\\Repair\\AddBruteForceCleanupJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddBruteForceCleanupJob.php',
+        'OC\\Repair\\AddCleanupBackgroundJobsJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupBackgroundJobsJob.php',
         'OC\\Repair\\AddCleanupDeletedUsersBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupDeletedUsersBackgroundJob.php',
         'OC\\Repair\\AddCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php',
         'OC\\Repair\\AddMetadataGenerationJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMetadataGenerationJob.php',

lib/private/Repair.php

@@ -9,6 +9,7 @@
 namespace OC;
 
 use OC\Repair\AddBruteForceCleanupJob;
+use OC\Repair\AddCleanupBackgroundJobsJob;
 use OC\Repair\AddCleanupDeletedUsersBackgroundJob;
 use OC\Repair\AddCleanupUpdaterBackupsJob;
 use OC\Repair\AddMetadataGenerationJob;
@@ -134,14 +135,14 @@ public function addStep(IRepairStep|string $repairStep, bool $includeExpensive =
 				}
 			}
 
-			if (!($s instanceof IRepairStep)) {
+			if (!$s instanceof IRepairStep) {
 				throw new \Exception("Repair step '$repairStep' is not of type \\OCP\\Migration\\IRepairStep");
 			}
 
 			$repairStep = $s;
 		}
 
-		if (($repairStep instanceof IRepairStepExpensive) && !$includeExpensive) {
+		if ($repairStep instanceof IRepairStepExpensive && !$includeExpensive) {
 			$this->debug("Skipping expensive repair step '" . $repairStep::class . "'");
 		} else {
 			$this->repairSteps[] = $repairStep;
@@ -195,6 +196,7 @@ public static function getRepairSteps(bool $includeExpensive = false): array {
 			Server::get(SanitizeAccountProperties::class),
 			Server::get(AddMovePreviewJob::class),
 			Server::get(ConfigKeyMigration::class),
+			Server::get(AddCleanupBackgroundJobsJob::class),
 		];
 
 		if ($includeExpensive) {

lib/private/Repair/AddCleanupBackgroundJobsJob.php

@@ -0,0 +1,33 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OC\Repair;
+
+use OC\Core\BackgroundJobs\CleanupBackgroundJobsJob;
+use OCP\BackgroundJob\IJobList;
+use OCP\Migration\IOutput;
+use OCP\Migration\IRepairStep;
+use Override;
+
+class AddCleanupBackgroundJobsJob implements IRepairStep {
+	public function __construct(
+		private readonly IJobList $jobList,
+	) {
+	}
+
+	#[\Override]
+	public function getName(): string {
+		return 'Cleanup completed background jobs';
+	}
+
+	#[Override]
+	public function run(IOutput $output): void {
+		$this->jobList->add(CleanupBackgroundJobsJob::class);
+	}
+}

lib/private/Setup.php

@@ -16,6 +16,7 @@
 use OC\AppFramework\Bootstrap\Coordinator;
 use OC\Authentication\Token\PublicKeyTokenProvider;
 use OC\Authentication\Token\TokenCleanupJob;
+use OC\Core\BackgroundJobs\CleanupBackgroundJobsJob;
 use OC\Core\BackgroundJobs\ExpirePreviewsJob;
 use OC\Core\BackgroundJobs\GenerateMetadataJob;
 use OC\Core\BackgroundJobs\PreviewMigrationJob;
@@ -532,6 +533,7 @@ public static function installBackgroundJobs(): void {
 		$jobList->add(GenerateMetadataJob::class);
 		$jobList->add(PreviewMigrationJob::class);
 		$jobList->add(ExpirePreviewsJob::class);
+		$jobList->add(CleanupBackgroundJobsJob::class);
 	}
 
 	/**

lib/private/Snowflake/SnowflakeGenerator.php

@@ -12,6 +12,7 @@
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\IConfig;
 use OCP\Snowflake\ISnowflakeGenerator;
+use OCP\Util;
 use Override;
 
 /**
@@ -108,16 +109,7 @@ private function getCurrentTime(): array {
 	 * @return int<0,511>
 	 */
 	private function getServerId(): int {
-		$serverid = $this->config->getSystemValueInt('serverid', -1);
-		if ($serverid < 1) {
-			// Fallback: generates a server ID based on hostname
-			// or random bytes if hostname isn't available
-			/** @var int<0,max> */
-			$serverid = hexdec(hash('xxh32', gethostname() ?: random_bytes(8)));
-		}
-
-		/** @var int<0,511> */
-		return $serverid & 0x1FF;
+		return Util::getServerId();
 	}
 
 	private function isCli(): bool {

lib/public/Util.php

@@ -279,6 +279,24 @@
 		return $host_name;
 	}
 
+	/**
+	 * Returns configured Server ID or use default fallback
+	 */
+	public static function getServerId() {
+		$config = Server::get(IConfig::class);
+
+		$serverid = $config->getSystemValueInt('serverid', -1);
+		if ($serverid < 1) {
+			// Fallback: generates a server ID based on hostname
+			// or random bytes if hostname isn't available
+			/** @var int<0,max> */
+			$serverid = hexdec(hash('xxh32', gethostname() ?: random_bytes(8)));
+		}
+
+		/** @var int<0,511> */
+		return $serverid & 0x1FF;
+	}
+
 	/**
 	 * Returns the default email address
 	 * @param string $user_part the user part of the address