Skip to content

Commit 6149168

Browse files
CarlSchwanCarlSchwan
committed
feat(preview): On demand preview migration
When requesting previews, which we don't find in oc_previews, search in IAppData first before creating them. Move the logic from MovepreviewJob to PreviewMigrationService and reuse that in the Preview Generator. At the same time rename MovePreviewJob to PreviewMigrationJob as it is a better name. Signed-off-by: Carl Schwan <carl.schwan@nextcloud.com>
1 parent bd90e7c commit 6149168

11 files changed

Lines changed: 278 additions & 125 deletions

File tree

‎core/BackgroundJobs/PreviewMigrationJob.php‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/*
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH
7+
* SPDX-FileContributor: Carl Schwan
8+
* SPDX-License-Identifier: AGPL-3.0-or-later
9+
*/
10+
11+
namespace OC\Core\BackgroundJobs;
12+
13+
use OC\Preview\Db\Preview;
14+
use OC\Preview\PreviewMigrationService;
15+
use OCP\AppFramework\Utility\ITimeFactory;
16+
use OCP\BackgroundJob\TimedJob;
17+
use OCP\DB\IResult;
18+
use OCP\Files\IRootFolder;
19+
use OCP\IAppConfig;
20+
use OCP\IConfig;
21+
use OCP\IDBConnection;
22+
use Override;
23+
24+
class PreviewMigrationJob extends TimedJob {
25+
private string $previewRootPath;
26+
27+
public function __construct(
28+
ITimeFactory $time,
29+
private readonly IAppConfig $appConfig,
30+
private readonly IConfig $config,
31+
private readonly IDBConnection $connection,
32+
private readonly IRootFolder $rootFolder,
33+
private readonly PreviewMigrationService $migrationService,
34+
) {
35+
parent::__construct($time);
36+
37+
$this->setTimeSensitivity(self::TIME_INSENSITIVE);
38+
$this->setInterval(24 * 60 * 60);
39+
$this->previewRootPath = 'appdata_' . $this->config->getSystemValueString('instanceid') . '/preview/';
40+
}
41+
42+
#[Override]
43+
protected function run(mixed $argument): void {
44+
if ($this->appConfig->getValueBool('core', 'previewMovedDone')) {
45+
return;
46+
}
47+
48+
$startTime = time();
49+
while (true) {
50+
$qb = $this->connection->getQueryBuilder();
51+
$qb->select('path')
52+
->from('filecache')
53+
// Hierarchical preview folder structure
54+
->where($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%/%/%/%/%/%/%/%')))
55+
// Legacy flat preview folder structure
56+
->orWhere($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%.%')))
57+
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
58+
->setMaxResults(100);
59+
60+
$result = $qb->executeQuery();
61+
$foundPreviews = $this->processQueryResult($result);
62+
63+
if (!$foundPreviews) {
64+
break;
65+
}
66+
67+
// Stop if execution time is more than one hour.
68+
if (time() - $startTime > 3600) {
69+
return;
70+
}
71+
}
72+
73+
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
74+
}
75+
76+
private function processQueryResult(IResult $result): bool {
77+
$foundPreview = false;
78+
$fileIds = [];
79+
$flatFileIds = [];
80+
while ($row = $result->fetch()) {
81+
$pathSplit = explode('/', $row['path']);
82+
assert(count($pathSplit) >= 2);
83+
$fileId = (int)$pathSplit[count($pathSplit) - 2];
84+
if (count($pathSplit) === 11) {
85+
// Hierarchical structure
86+
if (!in_array($fileId, $fileIds)) {
87+
$fileIds[] = $fileId;
88+
}
89+
} else {
90+
// Flat structure
91+
if (!in_array($fileId, $flatFileIds)) {
92+
$flatFileIds[] = $fileId;
93+
}
94+
}
95+
$foundPreview = true;
96+
}
97+
98+
foreach ($fileIds as $fileId) {
99+
$this->migrationService->migrateFileId($fileId, flatPath: false);
100+
}
101+
102+
foreach ($flatFileIds as $fileId) {
103+
$this->migrationService->migrateFileId($fileId, flatPath: true);
104+
}
105+
return $foundPreview;
106+
}
107+
}

‎lib/composer/composer/LICENSE‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
Copyright (c) Nils Adermann, Jordi Boggiano
32

43
Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -18,4 +17,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
1817
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
1918
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
2019
THE SOFTWARE.
21-

‎lib/composer/composer/autoload_classmap.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1299,7 +1299,7 @@
12991299
'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => $baseDir . '/core/BackgroundJobs/CleanupLoginFlowV2.php',
13001300
'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => $baseDir . '/core/BackgroundJobs/GenerateMetadataJob.php',
13011301
'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => $baseDir . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php',
1302-
'OC\\Core\\BackgroundJobs\\MovePreviewJob' => $baseDir . '/core/BackgroundJobs/MovePreviewJob.php',
1302+
'OC\\Core\\BackgroundJobs\\PreviewMigrationJob' => $baseDir . '/core/BackgroundJobs/PreviewMigrationJob.php',
13031303
'OC\\Core\\Command\\App\\Disable' => $baseDir . '/core/Command/App/Disable.php',
13041304
'OC\\Core\\Command\\App\\Enable' => $baseDir . '/core/Command/App/Enable.php',
13051305
'OC\\Core\\Command\\App\\GetPath' => $baseDir . '/core/Command/App/GetPath.php',
@@ -1977,6 +1977,7 @@
19771977
'OC\\Preview\\PNG' => $baseDir . '/lib/private/Preview/PNG.php',
19781978
'OC\\Preview\\Photoshop' => $baseDir . '/lib/private/Preview/Photoshop.php',
19791979
'OC\\Preview\\Postscript' => $baseDir . '/lib/private/Preview/Postscript.php',
1980+
'OC\\Preview\\PreviewMigrationService' => $baseDir . '/lib/private/Preview/PreviewMigrationService.php',
19801981
'OC\\Preview\\PreviewService' => $baseDir . '/lib/private/Preview/PreviewService.php',
19811982
'OC\\Preview\\ProviderV2' => $baseDir . '/lib/private/Preview/ProviderV2.php',
19821983
'OC\\Preview\\SGI' => $baseDir . '/lib/private/Preview/SGI.php',

‎lib/composer/composer/autoload_static.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1340,7 +1340,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
13401340
'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CleanupLoginFlowV2.php',
13411341
'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/GenerateMetadataJob.php',
13421342
'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php',
1343-
'OC\\Core\\BackgroundJobs\\MovePreviewJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/MovePreviewJob.php',
1343+
'OC\\Core\\BackgroundJobs\\PreviewMigrationJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/PreviewMigrationJob.php',
13441344
'OC\\Core\\Command\\App\\Disable' => __DIR__ . '/../../..' . '/core/Command/App/Disable.php',
13451345
'OC\\Core\\Command\\App\\Enable' => __DIR__ . '/../../..' . '/core/Command/App/Enable.php',
13461346
'OC\\Core\\Command\\App\\GetPath' => __DIR__ . '/../../..' . '/core/Command/App/GetPath.php',
@@ -2018,6 +2018,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
20182018
'OC\\Preview\\PNG' => __DIR__ . '/../../..' . '/lib/private/Preview/PNG.php',
20192019
'OC\\Preview\\Photoshop' => __DIR__ . '/../../..' . '/lib/private/Preview/Photoshop.php',
20202020
'OC\\Preview\\Postscript' => __DIR__ . '/../../..' . '/lib/private/Preview/Postscript.php',
2021+
'OC\\Preview\\PreviewMigrationService' => __DIR__ . '/../../..' . '/lib/private/Preview/PreviewMigrationService.php',
20212022
'OC\\Preview\\PreviewService' => __DIR__ . '/../../..' . '/lib/private/Preview/PreviewService.php',
20222023
'OC\\Preview\\ProviderV2' => __DIR__ . '/../../..' . '/lib/private/Preview/ProviderV2.php',
20232024
'OC\\Preview\\SGI' => __DIR__ . '/../../..' . '/lib/private/Preview/SGI.php',

‎lib/private/Preview/Generator.php‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
use OCP\Files\NotPermittedException;
1818
use OCP\Files\SimpleFS\InMemoryFile;
1919
use OCP\Files\SimpleFS\ISimpleFile;
20+
use OCP\IAppConfig;
2021
use OCP\IConfig;
2122
use OCP\IImage;
2223
use OCP\IPreview;
@@ -30,13 +31,15 @@ class Generator {
3031
public const SEMAPHORE_ID_NEW = 0x07ea;
3132

3233
public function __construct(
33-
private IConfig $config,
34-
private IPreview $previewManager,
35-
private GeneratorHelper $helper,
36-
private IEventDispatcher $eventDispatcher,
37-
private LoggerInterface $logger,
38-
private PreviewMapper $previewMapper,
39-
private StorageFactory $storageFactory,
34+
private readonly IConfig $config,
35+
private readonly IAppConfig $appConfig,
36+
private readonly IPreview $previewManager,
37+
private readonly GeneratorHelper $helper,
38+
private readonly IEventDispatcher $eventDispatcher,
39+
private readonly LoggerInterface $logger,
40+
private readonly PreviewMapper $previewMapper,
41+
private readonly StorageFactory $storageFactory,
42+
private readonly PreviewMigrationService $migrationService,
4043
) {
4144
}
4245

@@ -108,6 +111,10 @@ public function generatePreviews(File $file, array $specifications, ?string $mim
108111

109112
[$file->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$file->getId()]);
110113

114+
if (empty($previews)) {
115+
$previews = $this->migrateOldPreviews($file->getId());
116+
}
117+
111118
$previewVersion = null;
112119
if ($file instanceof IVersionedPreviewFile) {
113120
$previewVersion = $file->getPreviewVersion();
@@ -193,6 +200,21 @@ public function generatePreviews(File $file, array $specifications, ?string $mim
193200
return $previewFile;
194201
}
195202

203+
/**
204+
* @return array<string|int, string[]>
205+
*/
206+
private function migrateOldPreviews(int $fileId): array {
207+
if ($this->appConfig->getValueBool('core', 'previewMovedDone')) {
208+
return [];
209+
}
210+
211+
$previews = $this->migrationService->migrateFileId($fileId, flatPath: false);
212+
if (empty($previews)) {
213+
$previews = $this->migrationService->migrateFileId($fileId, flatPath: true);
214+
}
215+
return $previews;
216+
}
217+
196218
/**
197219
* Acquire a semaphore of the specified id and concurrency, blocking if necessary.
198220
* Return an identifier of the semaphore on success, which can be used to release it via

core/BackgroundJobs/MovePreviewJob.php renamed to lib/private/Preview/PreviewMigrationService.php

Lines changed: 25 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -2,130 +2,60 @@
22

33
declare(strict_types=1);
44

5-
/*
5+
/**
66
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH
77
* SPDX-FileContributor: Carl Schwan
88
* SPDX-License-Identifier: AGPL-3.0-or-later
99
*/
1010

11-
namespace OC\Core\BackgroundJobs;
11+
namespace OC\Preview;
1212

1313
use OC\Files\SimpleFS\SimpleFile;
1414
use OC\Preview\Db\Preview;
1515
use OC\Preview\Db\PreviewMapper;
1616
use OC\Preview\Storage\StorageFactory;
17-
use OCP\AppFramework\Utility\ITimeFactory;
18-
use OCP\BackgroundJob\TimedJob;
1917
use OCP\DB\Exception;
20-
use OCP\DB\IResult;
2118
use OCP\Files\AppData\IAppDataFactory;
2219
use OCP\Files\IAppData;
2320
use OCP\Files\IMimeTypeDetector;
2421
use OCP\Files\IMimeTypeLoader;
2522
use OCP\Files\IRootFolder;
26-
use OCP\IAppConfig;
23+
use OCP\Files\NotFoundException;
2724
use OCP\IConfig;
2825
use OCP\IDBConnection;
29-
use Override;
3026
use Psr\Log\LoggerInterface;
3127

32-
class MovePreviewJob extends TimedJob {
28+
class PreviewMigrationService {
3329
private IAppData $appData;
3430
private string $previewRootPath;
3531

3632
public function __construct(
37-
ITimeFactory $time,
38-
private readonly IAppConfig $appConfig,
3933
private readonly IConfig $config,
40-
private readonly PreviewMapper $previewMapper,
41-
private readonly StorageFactory $storageFactory,
42-
private readonly IDBConnection $connection,
4334
private readonly IRootFolder $rootFolder,
35+
private readonly LoggerInterface $logger,
4436
private readonly IMimeTypeDetector $mimeTypeDetector,
4537
private readonly IMimeTypeLoader $mimeTypeLoader,
46-
private readonly LoggerInterface $logger,
38+
private readonly IDBConnection $connection,
39+
private readonly PreviewMapper $previewMapper,
40+
private readonly StorageFactory $storageFactory,
4741
IAppDataFactory $appDataFactory,
4842
) {
49-
parent::__construct($time);
50-
5143
$this->appData = $appDataFactory->get('preview');
52-
$this->setTimeSensitivity(self::TIME_INSENSITIVE);
53-
$this->setInterval(24 * 60 * 60);
5444
$this->previewRootPath = 'appdata_' . $this->config->getSystemValueString('instanceid') . '/preview/';
5545
}
5646

57-
#[Override]
58-
protected function run(mixed $argument): void {
59-
if ($this->appConfig->getValueBool('core', 'previewMovedDone')) {
60-
return;
61-
}
62-
63-
$startTime = time();
64-
while (true) {
65-
$qb = $this->connection->getQueryBuilder();
66-
$qb->select('path')
67-
->from('filecache')
68-
// Hierarchical preview folder structure
69-
->where($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%/%/%/%/%/%/%/%')))
70-
// Legacy flat preview folder structure
71-
->orWhere($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%.%')))
72-
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
73-
->setMaxResults(100);
74-
75-
$result = $qb->executeQuery();
76-
$foundPreviews = $this->processQueryResult($result);
77-
78-
if (!$foundPreviews) {
79-
break;
80-
}
81-
82-
// Stop if execution time is more than one hour.
83-
if (time() - $startTime > 3600) {
84-
return;
85-
}
86-
}
87-
88-
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
89-
}
90-
91-
private function processQueryResult(IResult $result): bool {
92-
$foundPreview = false;
93-
$fileIds = [];
94-
$flatFileIds = [];
95-
while ($row = $result->fetchAssociative()) {
96-
$pathSplit = explode('/', $row['path']);
97-
assert(count($pathSplit) >= 2);
98-
$fileId = (int)$pathSplit[count($pathSplit) - 2];
99-
if (count($pathSplit) === 11) {
100-
// Hierarchical structure
101-
if (!in_array($fileId, $fileIds)) {
102-
$fileIds[] = $fileId;
103-
}
104-
} else {
105-
// Flat structure
106-
if (!in_array($fileId, $flatFileIds)) {
107-
$flatFileIds[] = $fileId;
108-
}
109-
}
110-
$foundPreview = true;
111-
}
112-
113-
foreach ($fileIds as $fileId) {
114-
$this->processPreviews($fileId, flatPath: false);
115-
}
116-
117-
foreach ($flatFileIds as $fileId) {
118-
$this->processPreviews($fileId, flatPath: true);
119-
}
120-
return $foundPreview;
121-
}
122-
12347
/**
12448
* @param array<string|int, string[]> $previewFolders
49+
* @return Preview[]
12550
*/
126-
private function processPreviews(int $fileId, bool $flatPath): void {
51+
public function migrateFileId(int $fileId, bool $flatPath): array {
52+
$previews = [];
12753
$internalPath = $this->getInternalFolder((string)$fileId, $flatPath);
128-
$folder = $this->appData->getFolder($internalPath);
54+
try {
55+
$folder = $this->appData->getFolder($internalPath);
56+
} catch (NotFoundException) {
57+
return [];
58+
}
12959

13060
/**
13161
* @var list<array{file: SimpleFile, preview: Preview}> $previewFiles
@@ -152,6 +82,10 @@ private function processPreviews(int $fileId, bool $flatPath): void {
15282
];
15383
}
15484

85+
if (empty($previewFiles)) {
86+
return $previews;
87+
}
88+
15589
$qb = $this->connection->getQueryBuilder();
15690
$qb->select('storage', 'etag', 'mimetype')
15791
->from('filecache')
@@ -194,6 +128,8 @@ private function processPreviews(int $fileId, bool $flatPath): void {
194128
$this->previewMapper->delete($preview);
195129
throw $e;
196130
}
131+
132+
$previews[] = $preview;
197133
}
198134
} else {
199135
// No matching fileId, delete preview
@@ -211,9 +147,11 @@ private function processPreviews(int $fileId, bool $flatPath): void {
211147
}
212148

213149
$this->deleteFolder($internalPath);
150+
151+
return $previews;
214152
}
215153

216-
public static function getInternalFolder(string $name, bool $flatPath): string {
154+
private static function getInternalFolder(string $name, bool $flatPath): string {
217155
if ($flatPath) {
218156
return $name;
219157
}

0 commit comments

Comments
 (0)