Skip to content

Commit e1eea9b

Browse files
AltahrimAltahrim
authored
Merge pull request #60972 from nextcloud/feat/clean_crashed_jobs
Job run history cleanup
2 parents 718dfd0 + dc5499a commit e1eea9b

17 files changed

Lines changed: 309 additions & 43 deletions

File tree

‎config/config.sample.php‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3002,4 +3002,12 @@
30023002
* Defaults to ``0``.
30033003
*/
30043004
'preview_expiration_days' => 0,
3005+
3006+
/**
3007+
* Delete job runs older than a certain number of days.
3008+
* Less than one day is not allowed.
3009+
*
3010+
* Defaults to ``60``.
3011+
*/
3012+
'background_jobs_expiration_days' => 60,
30053013
];

‎core/BackgroundJobs/CleanupBackgroundJobsJob.php‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OC\Core\BackgroundJobs;
11+
12+
use DateTimeImmutable;
13+
use OC\BackgroundJob\JobRuns;
14+
use OCP\AppFramework\Utility\ITimeFactory;
15+
use OCP\BackgroundJob\IJob;
16+
use OCP\BackgroundJob\JobStatus;
17+
use OCP\BackgroundJob\TimedJob;
18+
use OCP\IConfig;
19+
use OCP\IServerInfo;
20+
use Override;
21+
use Psr\Log\LoggerInterface;
22+
use RuntimeException;
23+
24+
class CleanupBackgroundJobsJob extends TimedJob {
25+
public function __construct(
26+
ITimeFactory $time,
27+
private readonly JobRuns $jobRuns,
28+
private readonly IServerInfo $serverInfo,
29+
private readonly IConfig $config,
30+
private readonly LoggerInterface $logger,
31+
) {
32+
parent::__construct($time);
33+
$this->setInterval(60 * 60);
34+
$this->setTimeSensitivity(IJob::TIME_SENSITIVE);
35+
}
36+
37+
#[Override]
38+
protected function run($argument): void {
39+
$this->reapCrashedJobs();
40+
$this->cleanOldestRuns();
41+
}
42+
43+
private function reapCrashedJobs(): void {
44+
$currentServerId = $this->serverInfo->getServerId();
45+
46+
foreach ($this->jobRuns->runningJobs(1000) as $job) {
47+
if ($job->serverId !== $currentServerId) {
48+
continue;
49+
}
50+
$output = [];
51+
$result = 0;
52+
exec('ps -p ' . escapeshellarg((string)$job->pid) . ' -o cmd', $output, $result);
53+
if (count($output) === 1 && current($output) === 'CMD' && $result === 1) {
54+
// Process doesn't exists anymore
55+
$maxDuration = (new DateTimeImmutable())->diff($job->startedAt);
56+
$maxDuration
57+
= ($maxDuration->days * 24 * 60 * 60 * 1000)
58+
+ ($maxDuration->h * 60 * 60 * 1000)
59+
+ ($maxDuration->i * 60 * 1000)
60+
+ ($maxDuration->s * 1000)
61+
+ (int)($maxDuration->f * 1000);
62+
$this->jobRuns->finished($job->runId, $maxDuration, 0, JobStatus::CRASHED);
63+
$this->logger->warning('No process matching PID {pid} found on server {serverId}. Job {runId} was marked as crashed', [
64+
'pid' => $job->pid,
65+
'serverId' => $job->serverId,
66+
'runId' => $job->runId,
67+
]);
68+
}
69+
}
70+
}
71+
72+
private function cleanOldestRuns(): void {
73+
$daysToKeep = $this->config->getSystemValueInt('background_jobs_expiration_days', 60);
74+
if ($daysToKeep < 1) {
75+
throw new RuntimeException('Invalid number of days');
76+
}
77+
$cleanBeforeTimestamp = time() - ($daysToKeep * 24 * 3600);
78+
79+
$cleanedJobs = $this->jobRuns->deleteBefore($cleanBeforeTimestamp);
80+
if ($cleanedJobs > 0) {
81+
$this->logger->info(
82+
'Cleanup of old background jobs. Number of jobs removed: ' . $cleanedJobs . 'Reason: older than ' . $daysToKeep . ' days.',
83+
);
84+
}
85+
}
86+
}

‎core/Command/Background/JobsHistory.php‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
use OC\BackgroundJob\JobRuns;
1313
use OC\Core\Command\Base;
1414
use OCP\BackgroundJob\JobStatus;
15-
use OCP\IConfig;
15+
use OCP\IServerInfo;
1616
use OCP\Util;
1717
use Override;
1818
use Symfony\Component\Console\Input\InputInterface;
@@ -23,7 +23,7 @@
2323
final class JobsHistory extends Base {
2424
public function __construct(
2525
private readonly JobRuns $jobRuns,
26-
private IConfig $config,
26+
private readonly IServerInfo $serverInfo,
2727
) {
2828
parent::__construct();
2929
}
@@ -75,7 +75,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
7575
private function formatLine(iterable $jobs): \Generator {
7676
$jobsInfo = [];
7777
$now = time();
78-
$currentServerId = $this->config->getSystemValueInt('serverid', -1);
78+
$currentServerId = $this->serverInfo->getServerId();
7979
foreach ($jobs as $job) {
8080
$status = match ($job->status) {
8181
JobStatus::RUNNING => 'Running',

‎core/Command/Background/RunningJobs.php‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
use OC\BackgroundJob\JobRuns;
1313
use OC\Core\Command\Base;
14-
use OCP\IConfig;
14+
use OCP\IServerInfo;
1515
use Override;
1616
use Symfony\Component\Console\Input\InputInterface;
1717
use Symfony\Component\Console\Input\InputOption;
@@ -20,7 +20,7 @@
2020
final class RunningJobs extends Base {
2121
public function __construct(
2222
private readonly JobRuns $jobRuns,
23-
private IConfig $config,
23+
private readonly IServerInfo $serverInfo,
2424
) {
2525
parent::__construct();
2626
}
@@ -60,7 +60,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
6060

6161
private function formatLine(iterable $jobs): \Generator {
6262
$now = time();
63-
$currentServerId = $this->config->getSystemValueInt('serverid', -1);
63+
$currentServerId = $this->serverInfo->getServerId();
6464
foreach ($jobs as $job) {
6565
yield [
6666
'Run ID' => $job->runId,

‎lib/composer/composer/autoload_classmap.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -651,6 +651,7 @@
651651
'OCP\\IRequest' => $baseDir . '/lib/public/IRequest.php',
652652
'OCP\\IRequestId' => $baseDir . '/lib/public/IRequestId.php',
653653
'OCP\\IServerContainer' => $baseDir . '/lib/public/IServerContainer.php',
654+
'OCP\\IServerInfo' => $baseDir . '/lib/public/IServerInfo.php',
654655
'OCP\\ISession' => $baseDir . '/lib/public/ISession.php',
655656
'OCP\\IStreamImage' => $baseDir . '/lib/public/IStreamImage.php',
656657
'OCP\\ITagManager' => $baseDir . '/lib/public/ITagManager.php',
@@ -1311,6 +1312,7 @@
13111312
'OC\\Core\\AppInfo\\ConfigLexicon' => $baseDir . '/core/AppInfo/ConfigLexicon.php',
13121313
'OC\\Core\\BackgroundJobs\\BackgroundCleanupUpdaterBackupsJob' => $baseDir . '/core/BackgroundJobs/BackgroundCleanupUpdaterBackupsJob.php',
13131314
'OC\\Core\\BackgroundJobs\\CheckForUserCertificates' => $baseDir . '/core/BackgroundJobs/CheckForUserCertificates.php',
1315+
'OC\\Core\\BackgroundJobs\\CleanupBackgroundJobsJob' => $baseDir . '/core/BackgroundJobs/CleanupBackgroundJobsJob.php',
13141316
'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => $baseDir . '/core/BackgroundJobs/CleanupLoginFlowV2.php',
13151317
'OC\\Core\\BackgroundJobs\\ExpirePreviewsJob' => $baseDir . '/core/BackgroundJobs/ExpirePreviewsJob.php',
13161318
'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => $baseDir . '/core/BackgroundJobs/GenerateMetadataJob.php',
@@ -2051,6 +2053,7 @@
20512053
'OC\\Repair' => $baseDir . '/lib/private/Repair.php',
20522054
'OC\\RepairException' => $baseDir . '/lib/private/RepairException.php',
20532055
'OC\\Repair\\AddBruteForceCleanupJob' => $baseDir . '/lib/private/Repair/AddBruteForceCleanupJob.php',
2056+
'OC\\Repair\\AddCleanupBackgroundJobsJob' => $baseDir . '/lib/private/Repair/AddCleanupBackgroundJobsJob.php',
20542057
'OC\\Repair\\AddCleanupDeletedUsersBackgroundJob' => $baseDir . '/lib/private/Repair/AddCleanupDeletedUsersBackgroundJob.php',
20552058
'OC\\Repair\\AddCleanupUpdaterBackupsJob' => $baseDir . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php',
20562059
'OC\\Repair\\AddMetadataGenerationJob' => $baseDir . '/lib/private/Repair/AddMetadataGenerationJob.php',
@@ -2173,6 +2176,7 @@
21732176
'OC\\Security\\VerificationToken\\VerificationToken' => $baseDir . '/lib/private/Security/VerificationToken/VerificationToken.php',
21742177
'OC\\Server' => $baseDir . '/lib/private/Server.php',
21752178
'OC\\ServerContainer' => $baseDir . '/lib/private/ServerContainer.php',
2179+
'OC\\ServerInfo' => $baseDir . '/lib/private/ServerInfo.php',
21762180
'OC\\ServerNotAvailableException' => $baseDir . '/lib/private/ServerNotAvailableException.php',
21772181
'OC\\ServiceUnavailableException' => $baseDir . '/lib/private/ServiceUnavailableException.php',
21782182
'OC\\Session\\CryptoSessionData' => $baseDir . '/lib/private/Session/CryptoSessionData.php',

‎lib/composer/composer/autoload_static.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -692,6 +692,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
692692
'OCP\\IRequest' => __DIR__ . '/../../..' . '/lib/public/IRequest.php',
693693
'OCP\\IRequestId' => __DIR__ . '/../../..' . '/lib/public/IRequestId.php',
694694
'OCP\\IServerContainer' => __DIR__ . '/../../..' . '/lib/public/IServerContainer.php',
695+
'OCP\\IServerInfo' => __DIR__ . '/../../..' . '/lib/public/IServerInfo.php',
695696
'OCP\\ISession' => __DIR__ . '/../../..' . '/lib/public/ISession.php',
696697
'OCP\\IStreamImage' => __DIR__ . '/../../..' . '/lib/public/IStreamImage.php',
697698
'OCP\\ITagManager' => __DIR__ . '/../../..' . '/lib/public/ITagManager.php',
@@ -1352,6 +1353,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
13521353
'OC\\Core\\AppInfo\\ConfigLexicon' => __DIR__ . '/../../..' . '/core/AppInfo/ConfigLexicon.php',
13531354
'OC\\Core\\BackgroundJobs\\BackgroundCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/BackgroundCleanupUpdaterBackupsJob.php',
13541355
'OC\\Core\\BackgroundJobs\\CheckForUserCertificates' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CheckForUserCertificates.php',
1356+
'OC\\Core\\BackgroundJobs\\CleanupBackgroundJobsJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CleanupBackgroundJobsJob.php',
13551357
'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CleanupLoginFlowV2.php',
13561358
'OC\\Core\\BackgroundJobs\\ExpirePreviewsJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/ExpirePreviewsJob.php',
13571359
'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/GenerateMetadataJob.php',
@@ -2092,6 +2094,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
20922094
'OC\\Repair' => __DIR__ . '/../../..' . '/lib/private/Repair.php',
20932095
'OC\\RepairException' => __DIR__ . '/../../..' . '/lib/private/RepairException.php',
20942096
'OC\\Repair\\AddBruteForceCleanupJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddBruteForceCleanupJob.php',
2097+
'OC\\Repair\\AddCleanupBackgroundJobsJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupBackgroundJobsJob.php',
20952098
'OC\\Repair\\AddCleanupDeletedUsersBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupDeletedUsersBackgroundJob.php',
20962099
'OC\\Repair\\AddCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php',
20972100
'OC\\Repair\\AddMetadataGenerationJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMetadataGenerationJob.php',
@@ -2214,6 +2217,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
22142217
'OC\\Security\\VerificationToken\\VerificationToken' => __DIR__ . '/../../..' . '/lib/private/Security/VerificationToken/VerificationToken.php',
22152218
'OC\\Server' => __DIR__ . '/../../..' . '/lib/private/Server.php',
22162219
'OC\\ServerContainer' => __DIR__ . '/../../..' . '/lib/private/ServerContainer.php',
2220+
'OC\\ServerInfo' => __DIR__ . '/../../..' . '/lib/private/ServerInfo.php',
22172221
'OC\\ServerNotAvailableException' => __DIR__ . '/../../..' . '/lib/private/ServerNotAvailableException.php',
22182222
'OC\\ServiceUnavailableException' => __DIR__ . '/../../..' . '/lib/private/ServiceUnavailableException.php',
22192223
'OC\\Session\\CryptoSessionData' => __DIR__ . '/../../..' . '/lib/private/Session/CryptoSessionData.php',

‎lib/private/BackgroundJob/JobRuns.php‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,18 @@ public function finished(int|string $runId, int $duration, int $memoryPeakUsage,
6262
return $result === 1;
6363
}
6464

65+
public function deleteBefore(int $timestamp): int {
66+
$beforeSnowflake = $this->snowflakeGenerator->minForTimeId($timestamp);
67+
$beforeSnowflake = '91480652934574081';
68+
$qb = $this->connection->getQueryBuilder();
69+
$result = $qb
70+
->delete(self::TABLE)
71+
->where($qb->expr()->lt('run_id', $qb->createNamedParameter($beforeSnowflake)))
72+
->executeStatement();
73+
74+
return $result;
75+
}
76+
6577
#[Override]
6678
public function runningJobs(int $limit = 200): \Generator {
6779
$qb = $this->connection->getQueryBuilder();

‎lib/private/Repair.php‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
namespace OC;
1010

1111
use OC\Repair\AddBruteForceCleanupJob;
12+
use OC\Repair\AddCleanupBackgroundJobsJob;
1213
use OC\Repair\AddCleanupDeletedUsersBackgroundJob;
1314
use OC\Repair\AddCleanupUpdaterBackupsJob;
1415
use OC\Repair\AddMetadataGenerationJob;
@@ -134,14 +135,14 @@ public function addStep(IRepairStep|string $repairStep, bool $includeExpensive =
134135
}
135136
}
136137

137-
if (!($s instanceof IRepairStep)) {
138+
if (!$s instanceof IRepairStep) {
138139
throw new \Exception("Repair step '$repairStep' is not of type \\OCP\\Migration\\IRepairStep");
139140
}
140141

141142
$repairStep = $s;
142143
}
143144

144-
if (($repairStep instanceof IRepairStepExpensive) && !$includeExpensive) {
145+
if ($repairStep instanceof IRepairStepExpensive && !$includeExpensive) {
145146
$this->debug("Skipping expensive repair step '" . $repairStep::class . "'");
146147
} else {
147148
$this->repairSteps[] = $repairStep;
@@ -195,6 +196,7 @@ public static function getRepairSteps(bool $includeExpensive = false): array {
195196
Server::get(SanitizeAccountProperties::class),
196197
Server::get(AddMovePreviewJob::class),
197198
Server::get(ConfigKeyMigration::class),
199+
Server::get(AddCleanupBackgroundJobsJob::class),
198200
];
199201

200202
if ($includeExpensive) {

‎lib/private/Repair/AddCleanupBackgroundJobsJob.php‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OC\Repair;
11+
12+
use OC\Core\BackgroundJobs\CleanupBackgroundJobsJob;
13+
use OCP\BackgroundJob\IJobList;
14+
use OCP\Migration\IOutput;
15+
use OCP\Migration\IRepairStep;
16+
use Override;
17+
18+
class AddCleanupBackgroundJobsJob implements IRepairStep {
19+
public function __construct(
20+
private readonly IJobList $jobList,
21+
) {
22+
}
23+
24+
#[\Override]
25+
public function getName(): string {
26+
return 'Cleanup completed background jobs';
27+
}
28+
29+
#[Override]
30+
public function run(IOutput $output): void {
31+
$this->jobList->add(CleanupBackgroundJobsJob::class);
32+
}
33+
}

‎lib/private/Server.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@
227227
use OCP\IRequest;
228228
use OCP\IRequestId;
229229
use OCP\IServerContainer;
230+
use OCP\IServerInfo;
230231
use OCP\ISession;
231232
use OCP\ITagManager;
232233
use OCP\ITempManager;
@@ -1146,6 +1147,9 @@ function () use ($c) {
11461147

11471148
return $c->get(FileSequence::class);
11481149
}, false);
1150+
$this->registerAlias(ISnowflakeDecoder::class, SnowflakeDecoder::class);
1151+
$this->registerAlias(IJobRuns::class, JobRuns::class);
1152+
$this->registerAlias(IServerInfo::class, ServerInfo::class);
11491153

11501154
$this->connectDispatcher();
11511155
}

0 commit comments

Comments
 (0)