Skip to content

Commit c2c8162

Browse files
CarlSchwanCarlSchwan
committed
perf(version): Batch propagator for multi-write operations
Signed-off-by: Carl Schwan <carl@carlschwan.eu>
1 parent defb6e8 commit c2c8162

1 file changed

Lines changed: 81 additions & 54 deletions

File tree

‎apps/files_versions/lib/Storage.php‎

Lines changed: 81 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
use OCP\Command\IBus;
2727
use OCP\EventDispatcher\IEventDispatcher;
2828
use OCP\Files;
29+
use OCP\Files\Cache\IPropagator;
2930
use OCP\Files\FileInfo;
3031
use OCP\Files\Folder;
3132
use OCP\Files\IMimeTypeDetector;
@@ -229,6 +230,14 @@ public static function markDeletedFile($path) {
229230
'filename' => $filename];
230231
}
231232

233+
private static function getUserStoragePropagator(string $uid): ?IPropagator {
234+
try {
235+
return Server::get(IRootFolder::class)->getUserFolder($uid)->getStorage()->getPropagator();
236+
} catch (\Exception) {
237+
return null;
238+
}
239+
}
240+
232241
/**
233242
* delete the version from the storage and cache
234243
*
@@ -259,10 +268,16 @@ public static function delete($path) {
259268

260269
$versions = self::getVersions($uid, $filename);
261270
if (!empty($versions)) {
262-
foreach ($versions as $v) {
263-
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
264-
self::deleteVersion($view, $filename . '.v' . $v['version']);
265-
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
271+
$propagator = self::getUserStoragePropagator($uid);
272+
$propagator?->beginBatch();
273+
try {
274+
foreach ($versions as $v) {
275+
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
276+
self::deleteVersion($view, $filename . '.v' . $v['version']);
277+
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
278+
}
279+
} finally {
280+
$propagator?->commitBatch();
266281
}
267282
}
268283
}
@@ -614,21 +629,27 @@ public static function expireOlderThanMaxForUser($uid) {
614629
return $version < $threshold;
615630
});
616631

617-
foreach ($versions as $version) {
618-
$internalPath = $version->getInternalPath();
619-
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
632+
$propagator = self::getUserStoragePropagator($uid);
633+
$propagator?->beginBatch();
634+
try {
635+
foreach ($versions as $version) {
636+
$internalPath = $version->getInternalPath();
637+
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
620638

621-
$versionEntity = isset($versionEntities[$version->getId()]) ? $versionEntities[$version->getId()] : null;
622-
if (!is_null($versionEntity)) {
623-
$versionsMapper->delete($versionEntity);
624-
}
639+
$versionEntity = isset($versionEntities[$version->getId()]) ? $versionEntities[$version->getId()] : null;
640+
if (!is_null($versionEntity)) {
641+
$versionsMapper->delete($versionEntity);
642+
}
625643

626-
try {
627-
$version->delete();
628-
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
629-
} catch (NotPermittedException $e) {
630-
Server::get(LoggerInterface::class)->error("Missing permissions to delete version for user {$uid}: {$internalPath}", ['app' => 'files_versions', 'exception' => $e]);
644+
try {
645+
$version->delete();
646+
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
647+
} catch (NotPermittedException $e) {
648+
Server::get(LoggerInterface::class)->error("Missing permissions to delete version for user {$uid}: {$internalPath}", ['app' => 'files_versions', 'exception' => $e]);
649+
}
631650
}
651+
} finally {
652+
$propagator?->commitBatch();
632653
}
633654
}
634655

@@ -929,47 +950,53 @@ public static function expire($filename, $uid) {
929950
$versionsSize = $versionsSize - $sizeOfDeletedVersions;
930951
}
931952

932-
foreach ($toDelete as $key => $path) {
933-
// Make sure to cleanup version table relations as expire does not pass deleteVersion
934-
try {
935-
/** @var VersionsMapper $versionsMapper */
936-
$versionsMapper = Server::get(VersionsMapper::class);
937-
$file = Server::get(IRootFolder::class)->getUserFolder($uid)->get($filename);
938-
$pathparts = pathinfo($path);
939-
$timestamp = (int)substr($pathparts['extension'] ?? '', 1);
940-
$versionEntity = $versionsMapper->findVersionForFileId($file->getId(), $timestamp);
941-
if ($versionEntity->getMetadataValue('label') !== null && $versionEntity->getMetadataValue('label') !== '') {
942-
continue;
953+
$propagator = self::getUserStoragePropagator($uid);
954+
$propagator?->beginBatch();
955+
try {
956+
foreach ($toDelete as $key => $path) {
957+
// Make sure to cleanup version table relations as expire does not pass deleteVersion
958+
try {
959+
/** @var VersionsMapper $versionsMapper */
960+
$versionsMapper = Server::get(VersionsMapper::class);
961+
$file = Server::get(IRootFolder::class)->getUserFolder($uid)->get($filename);
962+
$pathparts = pathinfo($path);
963+
$timestamp = (int)substr($pathparts['extension'] ?? '', 1);
964+
$versionEntity = $versionsMapper->findVersionForFileId($file->getId(), $timestamp);
965+
if ($versionEntity->getMetadataValue('label') !== null && $versionEntity->getMetadataValue('label') !== '') {
966+
continue;
967+
}
968+
$versionsMapper->delete($versionEntity);
969+
} catch (DoesNotExistException $e) {
943970
}
944-
$versionsMapper->delete($versionEntity);
945-
} catch (DoesNotExistException $e) {
946-
}
947971

948-
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
949-
self::deleteVersion($versionsFileview, $path);
950-
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
951-
unset($allVersions[$key]); // update array with the versions we keep
952-
$logger->info('Expire: ' . $path, ['app' => 'files_versions']);
953-
}
972+
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
973+
self::deleteVersion($versionsFileview, $path);
974+
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
975+
unset($allVersions[$key]); // update array with the versions we keep
976+
$logger->info('Expire: ' . $path, ['app' => 'files_versions']);
977+
}
954978

955-
// Check if enough space is available after versions are rearranged.
956-
// If not we delete the oldest versions until we meet the size limit for versions,
957-
// but always keep the two latest versions
958-
$numOfVersions = count($allVersions) - 2 ;
959-
$i = 0;
960-
// sort oldest first and make sure that we start at the first element
961-
ksort($allVersions);
962-
reset($allVersions);
963-
while ($availableSpace < 0 && $i < $numOfVersions) {
964-
$version = current($allVersions);
965-
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
966-
self::deleteVersion($versionsFileview, $version['path'] . '.v' . $version['version']);
967-
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
968-
$logger->info('running out of space! Delete oldest version: ' . $version['path'] . '.v' . $version['version'], ['app' => 'files_versions']);
969-
$versionsSize -= $version['size'];
970-
$availableSpace += $version['size'];
971-
next($allVersions);
972-
$i++;
979+
// Check if enough space is available after versions are rearranged.
980+
// If not we delete the oldest versions until we meet the size limit for versions,
981+
// but always keep the two latest versions
982+
$numOfVersions = count($allVersions) - 2 ;
983+
$i = 0;
984+
// sort oldest first and make sure that we start at the first element
985+
ksort($allVersions);
986+
reset($allVersions);
987+
while ($availableSpace < 0 && $i < $numOfVersions) {
988+
$version = current($allVersions);
989+
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
990+
self::deleteVersion($versionsFileview, $version['path'] . '.v' . $version['version']);
991+
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
992+
$logger->info('running out of space! Delete oldest version: ' . $version['path'] . '.v' . $version['version'], ['app' => 'files_versions']);
993+
$versionsSize -= $version['size'];
994+
$availableSpace += $version['size'];
995+
next($allVersions);
996+
$i++;
997+
}
998+
} finally {
999+
$propagator?->commitBatch();
9731000
}
9741001

9751002
return $versionsSize; // finally return the new size of the version history

0 commit comments

Comments
 (0)