Skip to content

Commit 4783459

Browse files
st3inyst3iny
authored
Merge pull request #52589 from nextcloud/fix/dav/orphan-cleanup-job
fix(dav): move orphan cleaning logic to a chunked background job
2 parents 5644f33 + d0fcb7e commit 4783459

5 files changed

Lines changed: 287 additions & 62 deletions

File tree

‎apps/dav/composer/composer/autoload_classmap.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => $baseDir . '/../lib/BackgroundJob/CalendarRetentionJob.php',
1717
'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => $baseDir . '/../lib/BackgroundJob/CleanupDirectLinksJob.php',
1818
'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => $baseDir . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php',
19+
'OCA\\DAV\\BackgroundJob\\CleanupOrphanedChildrenJob' => $baseDir . '/../lib/BackgroundJob/CleanupOrphanedChildrenJob.php',
1920
'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => $baseDir . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php',
2021
'OCA\\DAV\\BackgroundJob\\EventReminderJob' => $baseDir . '/../lib/BackgroundJob/EventReminderJob.php',
2122
'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',

‎apps/dav/composer/composer/autoload_static.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ class ComposerStaticInitDAV
3131
'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CalendarRetentionJob.php',
3232
'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupDirectLinksJob.php',
3333
'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php',
34+
'OCA\\DAV\\BackgroundJob\\CleanupOrphanedChildrenJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupOrphanedChildrenJob.php',
3435
'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => __DIR__ . '/..' . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php',
3536
'OCA\\DAV\\BackgroundJob\\EventReminderJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/EventReminderJob.php',
3637
'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',

‎apps/dav/lib/BackgroundJob/CleanupOrphanedChildrenJob.php‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OCA\DAV\BackgroundJob;
11+
12+
use OCA\DAV\CalDAV\CalDavBackend;
13+
use OCP\AppFramework\Utility\ITimeFactory;
14+
use OCP\BackgroundJob\IJobList;
15+
use OCP\BackgroundJob\QueuedJob;
16+
use OCP\DB\QueryBuilder\IQueryBuilder;
17+
use OCP\IDBConnection;
18+
use Psr\Log\LoggerInterface;
19+
20+
class CleanupOrphanedChildrenJob extends QueuedJob {
21+
public const ARGUMENT_CHILD_TABLE = 'childTable';
22+
public const ARGUMENT_PARENT_TABLE = 'parentTable';
23+
public const ARGUMENT_PARENT_ID = 'parentId';
24+
public const ARGUMENT_LOG_MESSAGE = 'logMessage';
25+
26+
private const BATCH_SIZE = 1000;
27+
28+
public function __construct(
29+
ITimeFactory $time,
30+
private readonly IDBConnection $connection,
31+
private readonly LoggerInterface $logger,
32+
private readonly IJobList $jobList,
33+
) {
34+
parent::__construct($time);
35+
}
36+
37+
protected function run($argument): void {
38+
$childTable = $argument[self::ARGUMENT_CHILD_TABLE];
39+
$parentTable = $argument[self::ARGUMENT_PARENT_TABLE];
40+
$parentId = $argument[self::ARGUMENT_PARENT_ID];
41+
$logMessage = $argument[self::ARGUMENT_LOG_MESSAGE];
42+
43+
$orphanCount = $this->cleanUpOrphans($childTable, $parentTable, $parentId);
44+
$this->logger->debug(sprintf($logMessage, $orphanCount));
45+
46+
// Requeue if there might be more orphans
47+
if ($orphanCount >= self::BATCH_SIZE) {
48+
$this->jobList->add(self::class, $argument);
49+
}
50+
}
51+
52+
private function cleanUpOrphans(
53+
string $childTable,
54+
string $parentTable,
55+
string $parentId,
56+
): int {
57+
// We can't merge both queries into a single one here as DELETEing from a table while
58+
// SELECTing it in a sub query is not supported by Oracle DB.
59+
// Ref https://docs.oracle.com/cd/E17952_01/mysql-8.0-en/delete.html#idm46006185488144
60+
61+
$selectQb = $this->connection->getQueryBuilder();
62+
63+
$selectQb->select('c.id')
64+
->from($childTable, 'c')
65+
->leftJoin('c', $parentTable, 'p', $selectQb->expr()->eq('c.' . $parentId, 'p.id'))
66+
->where($selectQb->expr()->isNull('p.id'))
67+
->setMaxResults(self::BATCH_SIZE);
68+
69+
if (\in_array($parentTable, ['calendars', 'calendarsubscriptions'], true)) {
70+
$calendarType = $parentTable === 'calendarsubscriptions' ? CalDavBackend::CALENDAR_TYPE_SUBSCRIPTION : CalDavBackend::CALENDAR_TYPE_CALENDAR;
71+
$selectQb->andWhere($selectQb->expr()->eq('c.calendartype', $selectQb->createNamedParameter($calendarType, IQueryBuilder::PARAM_INT), IQueryBuilder::PARAM_INT));
72+
}
73+
74+
$result = $selectQb->executeQuery();
75+
$rows = $result->fetchAll();
76+
$result->closeCursor();
77+
if (empty($rows)) {
78+
return 0;
79+
}
80+
81+
$orphanItems = array_map(static fn ($row) => $row['id'], $rows);
82+
$deleteQb = $this->connection->getQueryBuilder();
83+
$deleteQb->delete($childTable)
84+
->where($deleteQb->expr()->in('id', $deleteQb->createNamedParameter($orphanItems, IQueryBuilder::PARAM_INT_ARRAY)));
85+
$deleteQb->executeStatement();
86+
87+
return count($orphanItems);
88+
}
89+
}

‎apps/dav/lib/Migration/RemoveOrphanEventsAndContacts.php‎

Lines changed: 25 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -8,82 +8,45 @@
88
*/
99
namespace OCA\DAV\Migration;
1010

11-
use OCA\DAV\CalDAV\CalDavBackend;
12-
use OCP\DB\QueryBuilder\IQueryBuilder;
13-
use OCP\IDBConnection;
11+
use OCA\DAV\BackgroundJob\CleanupOrphanedChildrenJob;
12+
use OCP\BackgroundJob\IJobList;
1413
use OCP\Migration\IOutput;
1514
use OCP\Migration\IRepairStep;
1615

1716
class RemoveOrphanEventsAndContacts implements IRepairStep {
18-
1917
public function __construct(
20-
private IDBConnection $connection,
18+
private readonly IJobList $jobList,
2119
) {
2220
}
2321

24-
/**
25-
* @inheritdoc
26-
*/
2722
public function getName(): string {
28-
return 'Clean up orphan event and contact data';
23+
return 'Queue jobs to clean up orphan event and contact data';
2924
}
3025

31-
/**
32-
* @inheritdoc
33-
*/
34-
public function run(IOutput $output) {
35-
$orphanItems = $this->removeOrphanChildren('calendarobjects', 'calendars', 'calendarid');
36-
$output->info(sprintf('%d events without a calendar have been cleaned up', $orphanItems));
37-
$orphanItems = $this->removeOrphanChildren('calendarobjects_props', 'calendarobjects', 'objectid');
38-
$output->info(sprintf('%d properties without an events have been cleaned up', $orphanItems));
39-
$orphanItems = $this->removeOrphanChildren('calendarchanges', 'calendars', 'calendarid');
40-
$output->info(sprintf('%d changes without a calendar have been cleaned up', $orphanItems));
26+
public function run(IOutput $output): void {
27+
$this->queueJob('calendarobjects', 'calendars', 'calendarid', '%d events without a calendar have been cleaned up');
28+
$this->queueJob('calendarobjects_props', 'calendarobjects', 'objectid', '%d properties without an events have been cleaned up');
29+
$this->queueJob('calendarchanges', 'calendars', 'calendarid', '%d changes without a calendar have been cleaned up');
4130

42-
$orphanItems = $this->removeOrphanChildren('calendarobjects', 'calendarsubscriptions', 'calendarid');
43-
$output->info(sprintf('%d cached events without a calendar subscription have been cleaned up', $orphanItems));
44-
$orphanItems = $this->removeOrphanChildren('calendarchanges', 'calendarsubscriptions', 'calendarid');
45-
$output->info(sprintf('%d changes without a calendar subscription have been cleaned up', $orphanItems));
31+
$this->queueJob('calendarobjects', 'calendarsubscriptions', 'calendarid', '%d cached events without a calendar subscription have been cleaned up');
32+
$this->queueJob('calendarchanges', 'calendarsubscriptions', 'calendarid', '%d changes without a calendar subscription have been cleaned up');
4633

47-
$orphanItems = $this->removeOrphanChildren('cards', 'addressbooks', 'addressbookid');
48-
$output->info(sprintf('%d contacts without an addressbook have been cleaned up', $orphanItems));
49-
$orphanItems = $this->removeOrphanChildren('cards_properties', 'cards', 'cardid');
50-
$output->info(sprintf('%d properties without a contact have been cleaned up', $orphanItems));
51-
$orphanItems = $this->removeOrphanChildren('addressbookchanges', 'addressbooks', 'addressbookid');
52-
$output->info(sprintf('%d changes without an addressbook have been cleaned up', $orphanItems));
34+
$this->queueJob('cards', 'addressbooks', 'addressbookid', '%d contacts without an addressbook have been cleaned up');
35+
$this->queueJob('cards_properties', 'cards', 'cardid', '%d properties without a contact have been cleaned up');
36+
$this->queueJob('addressbookchanges', 'addressbooks', 'addressbookid', '%d changes without an addressbook have been cleaned up');
5337
}
5438

55-
protected function removeOrphanChildren($childTable, $parentTable, $parentId): int {
56-
$qb = $this->connection->getQueryBuilder();
57-
58-
$qb->select('c.id')
59-
->from($childTable, 'c')
60-
->leftJoin('c', $parentTable, 'p', $qb->expr()->eq('c.' . $parentId, 'p.id'))
61-
->where($qb->expr()->isNull('p.id'));
62-
63-
if (\in_array($parentTable, ['calendars', 'calendarsubscriptions'], true)) {
64-
$calendarType = $parentTable === 'calendarsubscriptions' ? CalDavBackend::CALENDAR_TYPE_SUBSCRIPTION : CalDavBackend::CALENDAR_TYPE_CALENDAR;
65-
$qb->andWhere($qb->expr()->eq('c.calendartype', $qb->createNamedParameter($calendarType, IQueryBuilder::PARAM_INT), IQueryBuilder::PARAM_INT));
66-
}
67-
68-
$result = $qb->executeQuery();
69-
70-
$orphanItems = [];
71-
while ($row = $result->fetch()) {
72-
$orphanItems[] = (int)$row['id'];
73-
}
74-
$result->closeCursor();
75-
76-
if (!empty($orphanItems)) {
77-
$qb->delete($childTable)
78-
->where($qb->expr()->in('id', $qb->createParameter('ids')));
79-
80-
$orphanItemsBatch = array_chunk($orphanItems, 1000);
81-
foreach ($orphanItemsBatch as $items) {
82-
$qb->setParameter('ids', $items, IQueryBuilder::PARAM_INT_ARRAY);
83-
$qb->executeStatement();
84-
}
85-
}
86-
87-
return count($orphanItems);
39+
private function queueJob(
40+
string $childTable,
41+
string $parentTable,
42+
string $parentId,
43+
string $logMessage,
44+
): void {
45+
$this->jobList->add(CleanupOrphanedChildrenJob::class, [
46+
CleanupOrphanedChildrenJob::ARGUMENT_CHILD_TABLE => $childTable,
47+
CleanupOrphanedChildrenJob::ARGUMENT_PARENT_TABLE => $parentTable,
48+
CleanupOrphanedChildrenJob::ARGUMENT_PARENT_ID => $parentId,
49+
CleanupOrphanedChildrenJob::ARGUMENT_LOG_MESSAGE => $logMessage,
50+
]);
8851
}
8952
}

‎apps/dav/tests/unit/BackgroundJob/CleanupOrphanedChildrenJobTest.php‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OCA\DAV\Tests\unit\BackgroundJob;
11+
12+
use OCA\DAV\BackgroundJob\CleanupOrphanedChildrenJob;
13+
use OCP\AppFramework\Utility\ITimeFactory;
14+
use OCP\BackgroundJob\IJobList;
15+
use OCP\DB\IResult;
16+
use OCP\DB\QueryBuilder\IExpressionBuilder;
17+
use OCP\DB\QueryBuilder\IQueryBuilder;
18+
use OCP\IDBConnection;
19+
use PHPUnit\Framework\MockObject\MockObject;
20+
use Psr\Log\LoggerInterface;
21+
use Test\TestCase;
22+
23+
class CleanupOrphanedChildrenJobTest extends TestCase {
24+
private CleanupOrphanedChildrenJob $job;
25+
26+
private ITimeFactory&MockObject $timeFactory;
27+
private IDBConnection&MockObject $connection;
28+
private LoggerInterface&MockObject $logger;
29+
private IJobList&MockObject $jobList;
30+
31+
protected function setUp(): void {
32+
parent::setUp();
33+
34+
$this->timeFactory = $this->createMock(ITimeFactory::class);
35+
$this->connection = $this->createMock(IDBConnection::class);
36+
$this->logger = $this->createMock(LoggerInterface::class);
37+
$this->jobList = $this->createMock(IJobList::class);
38+
39+
$this->job = new CleanupOrphanedChildrenJob(
40+
$this->timeFactory,
41+
$this->connection,
42+
$this->logger,
43+
$this->jobList,
44+
);
45+
}
46+
47+
private function getArgument(): array {
48+
return [
49+
'childTable' => 'childTable',
50+
'parentTable' => 'parentTable',
51+
'parentId' => 'parentId',
52+
'logMessage' => 'logMessage',
53+
];
54+
}
55+
56+
private function getMockQueryBuilder(): IQueryBuilder&MockObject {
57+
$expr = $this->createMock(IExpressionBuilder::class);
58+
$qb = $this->createMock(IQueryBuilder::class);
59+
$qb->method('select')
60+
->willReturnSelf();
61+
$qb->method('from')
62+
->willReturnSelf();
63+
$qb->method('leftJoin')
64+
->willReturnSelf();
65+
$qb->method('where')
66+
->willReturnSelf();
67+
$qb->method('setMaxResults')
68+
->willReturnSelf();
69+
$qb->method('andWhere')
70+
->willReturnSelf();
71+
$qb->method('expr')
72+
->willReturn($expr);
73+
$qb->method('delete')
74+
->willReturnSelf();
75+
return $qb;
76+
}
77+
78+
public function testRunWithoutOrphans(): void {
79+
$argument = $this->getArgument();
80+
$selectQb = $this->getMockQueryBuilder();
81+
$result = $this->createMock(IResult::class);
82+
83+
$this->connection->expects(self::once())
84+
->method('getQueryBuilder')
85+
->willReturn($selectQb);
86+
$selectQb->expects(self::once())
87+
->method('executeQuery')
88+
->willReturn($result);
89+
$result->expects(self::once())
90+
->method('fetchAll')
91+
->willReturn([]);
92+
$result->expects(self::once())
93+
->method('closeCursor');
94+
$this->jobList->expects(self::never())
95+
->method('add');
96+
97+
self::invokePrivate($this->job, 'run', [$argument]);
98+
}
99+
100+
public function testRunWithPartialBatch(): void {
101+
$argument = $this->getArgument();
102+
$selectQb = $this->getMockQueryBuilder();
103+
$deleteQb = $this->getMockQueryBuilder();
104+
$result = $this->createMock(IResult::class);
105+
106+
$qbInvocationCount = self::exactly(2);
107+
$this->connection->expects($qbInvocationCount)
108+
->method('getQueryBuilder')
109+
->willReturnCallback(function () use ($qbInvocationCount, $selectQb, $deleteQb) {
110+
return match ($qbInvocationCount->getInvocationCount()) {
111+
1 => $selectQb,
112+
2 => $deleteQb,
113+
};
114+
});
115+
$selectQb->expects(self::once())
116+
->method('executeQuery')
117+
->willReturn($result);
118+
$result->expects(self::once())
119+
->method('fetchAll')
120+
->willReturn([
121+
['id' => 42],
122+
['id' => 43],
123+
]);
124+
$result->expects(self::once())
125+
->method('closeCursor');
126+
$deleteQb->expects(self::once())
127+
->method('delete')
128+
->willReturnSelf();
129+
$deleteQb->expects(self::once())
130+
->method('executeStatement');
131+
$this->jobList->expects(self::never())
132+
->method('add');
133+
134+
self::invokePrivate($this->job, 'run', [$argument]);
135+
}
136+
137+
public function testRunWithFullBatch(): void {
138+
$argument = $this->getArgument();
139+
$selectQb = $this->getMockQueryBuilder();
140+
$deleteQb = $this->getMockQueryBuilder();
141+
$result = $this->createMock(IResult::class);
142+
143+
$qbInvocationCount = self::exactly(2);
144+
$this->connection->expects($qbInvocationCount)
145+
->method('getQueryBuilder')
146+
->willReturnCallback(function () use ($qbInvocationCount, $selectQb, $deleteQb) {
147+
return match ($qbInvocationCount->getInvocationCount()) {
148+
1 => $selectQb,
149+
2 => $deleteQb,
150+
};
151+
});
152+
$selectQb->expects(self::once())
153+
->method('executeQuery')
154+
->willReturn($result);
155+
$result->expects(self::once())
156+
->method('fetchAll')
157+
->willReturn(array_map(static fn ($i) => ['id' => 42 + $i], range(0, 999)));
158+
$result->expects(self::once())
159+
->method('closeCursor');
160+
$deleteQb->expects(self::once())
161+
->method('delete')
162+
->willReturnSelf();
163+
$deleteQb->expects(self::once())
164+
->method('executeStatement');
165+
$this->jobList->expects(self::once())
166+
->method('add')
167+
->with(CleanupOrphanedChildrenJob::class, $argument);
168+
169+
self::invokePrivate($this->job, 'run', [$argument]);
170+
}
171+
}

0 commit comments

Comments
 (0)