Skip to content

perf(files/cache): SQL aggregation for folder size calculation #59586

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
solracsf wants to merge 1 commit into
base: master
Choose a base branch
from
Open

perf(files/cache): SQL aggregation for folder size calculation #59586

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,4 +102,14 @@ public function least($x, $y): IQueryFunction {
public function now(): IQueryFunction {
return new QueryFunction('NOW()');
}

#[Override]
public function caseWhen($condition, $then, $else): IQueryFunction {
return new QueryFunction(
'CASE WHEN ' . $condition
. ' THEN ' . $this->helper->quoteColumnName($then)
. ' ELSE ' . $this->helper->quoteColumnName($else)
. ' END'
);
}
}
57 changes: 26 additions & 31 deletions lib/private/Files/Cache/Cache.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1015,7 +1015,6 @@ public function calculateFolderSize($path, $entry = null) {
/**
* inner function because we can't add new params to the public function without breaking any child classes
*
* @param string $path
* @param array|null|ICacheEntry $entry (optional) meta data of the folder
* @param bool $ignoreUnknown don't mark the folder size as unknown if any of it's children are unknown
* @return int|float
Expand All @@ -1029,7 +1028,17 @@ protected function calculateFolderSizeInner(string $path, $entry = null, bool $i
$id = $entry['fileid'];

$query = $this->getQueryBuilder();
$query->select('size', 'unencrypted_size')
// $effectiveSize is reused in both aggregates; Doctrine handles the duplicated parameter correctly
$effectiveSize = $query->func()->caseWhen(
$query->expr()->gt('unencrypted_size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)),
'unencrypted_size',
'size'
);
$query->selectAlias($query->func()->sum('size'), 'size_sum')
->selectAlias($query->func()->min('size'), 'size_min')
->selectAlias($query->func()->max('unencrypted_size'), 'unencrypted_max')
->selectAlias($query->func()->sum($effectiveSize), 'unencrypted_sum')
->selectAlias($query->func()->min($effectiveSize), 'unencrypted_min')
->from('filecache')
->whereStorageId($this->getNumericStorageId())
->whereParent($id);
Expand All @@ -1038,34 +1047,19 @@ protected function calculateFolderSizeInner(string $path, $entry = null, bool $i
}

$result = $query->executeQuery();
$rows = $result->fetchAll();
$agg = $result->fetch();
$result->closeCursor();

if ($rows) {
$sizes = array_map(function (array $row) {
return Util::numericToNumber($row['size']);
}, $rows);
$unencryptedOnlySizes = array_map(function (array $row) {
return Util::numericToNumber($row['unencrypted_size']);
}, $rows);
$unencryptedSizes = array_map(function (array $row) {
return Util::numericToNumber(($row['unencrypted_size'] > 0) ? $row['unencrypted_size'] : $row['size']);
}, $rows);

$sum = array_sum($sizes);
$min = min($sizes);

$unencryptedSum = array_sum($unencryptedSizes);
$unencryptedMin = min($unencryptedSizes);
$unencryptedMax = max($unencryptedOnlySizes);

$sum = 0 + $sum;
$min = 0 + $min;
if ($min === -1) {
$totalSize = $min;
} else {
$totalSize = $sum;
}
// SUM() returns NULL on empty set
if ($agg && $agg['size_sum'] !== null) {
$sum = Util::numericToNumber($agg['size_sum']);
$min = Util::numericToNumber($agg['size_min']);
$unencryptedMax = Util::numericToNumber($agg['unencrypted_max'] ?? 0);
$unencryptedSum = Util::numericToNumber($agg['unencrypted_sum'] ?? 0);
$unencryptedMin = Util::numericToNumber($agg['unencrypted_min'] ?? 0);

$totalSize = ($min === -1) ? $min : $sum;

if ($unencryptedMin === -1 || $min === -1) {
$unencryptedTotal = $unencryptedMin;
} else {
Expand All @@ -1077,15 +1071,16 @@ protected function calculateFolderSizeInner(string $path, $entry = null, bool $i
$unencryptedMax = 0;
}

// only set unencrypted size for a folder if any child entries have it set, or the folder is empty
// only set unencrypted size for a folder if any child entries have it set
// or if the folder is empty
$shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || ($entry['unencrypted_size'] ?? 0) > 0;
if ($entry['size'] !== $totalSize || (($entry['unencrypted_size'] ?? 0) !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) {
if ($shouldWriteUnEncryptedSize) {
// if all children have an unencrypted size of 0, just set the folder unencrypted size to 0 instead of summing the sizes
// if all children have an unencrypted size of 0
// just set the folder unencrypted size to 0 instead of summing the sizes
if ($unencryptedMax === 0) {
$unencryptedTotal = 0;
}

$this->update($id, [
'size' => $totalSize,
'unencrypted_size' => $unencryptedTotal,
Expand Down
11 changes: 11 additions & 0 deletions lib/public/DB/QueryBuilder/IFunctionBuilder.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,4 +176,15 @@ public function least($x, $y): IQueryFunction;
* @since 34.0.0
*/
public function now(): IQueryFunction;

/**
* Builds a simple CASE WHEN … THEN … ELSE … END expression.
*
* @param string|IQueryFunction $condition SQL condition (e.g. from `$expr->gt(…)`)
* @param string|ILiteral|IParameter|IQueryFunction $then Result when condition is true
* @param string|ILiteral|IParameter|IQueryFunction $else Result when condition is false
* @return IQueryFunction
* @since 34.0.0
*/
public function caseWhen($condition, $then, $else): IQueryFunction;
}
56 changes: 56 additions & 0 deletions tests/lib/DB/QueryBuilder/FunctionBuilderTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,4 +486,60 @@ public function testLeast(): void {
$result->closeCursor();
$this->assertEquals(1, $row);
}

public function testCaseWhenTrue(): void {
$query = $this->connection->getQueryBuilder();

$caseExpression = $query->func()->caseWhen(
$query->expr()->gt(new Literal(5), new Literal(3)),
new Literal("'yes'"),
new Literal("'no'")
);
$query->select($caseExpression);
$query->from('appconfig')
->setMaxResults(1);

$result = $query->executeQuery();
$row = $result->fetchOne();
$result->closeCursor();
$this->assertEquals('yes', $row);
}

public function testCaseWhenFalse(): void {
$query = $this->connection->getQueryBuilder();

$caseExpression = $query->func()->caseWhen(
$query->expr()->gt(new Literal(1), new Literal(3)),
new Literal("'yes'"),
new Literal("'no'")
);
$query->select($caseExpression);
$query->from('appconfig')
->setMaxResults(1);

$result = $query->executeQuery();
$row = $result->fetchOne();
$result->closeCursor();
$this->assertEquals('no', $row);
}

public function testCaseWhenWithAggregate(): void {
$this->addIntDummyData(); // inserts editable = 1, 2, 3

$query = $this->connection->getQueryBuilder();

$effective = $query->func()->caseWhen(
$query->expr()->neq('editable', $query->createNamedParameter(1, IQueryBuilder::PARAM_INT)),
new Literal(1),
new Literal(50)
);
$query->select($query->func()->min($effective));
$query->from('systemtag')
->where($query->expr()->eq('name', $query->createNamedParameter('group_concat')));

$result = $query->executeQuery();
$row = $result->fetchOne();
$result->closeCursor();
$this->assertEquals(1, $row);
}
}
Loading