Skip to content

Commit e497d0b

Browse files
KocKoc
committed
Fix: Large table causes sql error (cache cells)
Signed-off-by: Kostiantyn Miakshyn <molodchick@gmail.com>
1 parent b8f18e9 commit e497d0b

File tree

7 files changed

+161
-15
lines changed

7 files changed

+161
-15
lines changed

‎appinfo/info.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ Have a good time and manage whatever you want.
5757
<post-migration>
5858
<step>OCA\Tables\Migration\NewDbStructureRepairStep</step>
5959
<step>OCA\Tables\Migration\DbRowSleeveSequence</step>
60+
<step>OCA\Tables\Migration\CacheSleeveCells</step>
6061
</post-migration>
6162
</repair-steps>
6263
<commands>

‎lib/Db/Row2Mapper.php‎

Lines changed: 34 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -731,9 +731,11 @@ public function insert(Row2 $row): Row2 {
731731
}
732732

733733
// write all cells to its db-table
734+
$cells = [];
734735
foreach ($row->getData() as $cell) {
735-
$this->insertCell($rowSleeve->getId(), $cell['columnId'], $cell['value'], $rowSleeve->getLastEditAt(), $rowSleeve->getLastEditBy());
736+
$cells[$cell['columnId']] = $this->insertCell($rowSleeve->getId(), $cell['columnId'], $cell['value'], $rowSleeve->getLastEditAt(), $rowSleeve->getLastEditBy());
736737
}
738+
$rowSleeve->setCells(json_encode($cells));
737739

738740
return $row;
739741
}
@@ -750,9 +752,9 @@ public function update(Row2 $row): Row2 {
750752

751753
// update meta data for sleeve
752754
try {
753-
$sleeve = $this->rowSleeveMapper->find($row->getId());
754-
$this->updateMetaData($sleeve);
755-
$this->rowSleeveMapper->update($sleeve);
755+
$rowSleeve = $this->rowSleeveMapper->find($row->getId());
756+
$this->updateMetaData($rowSleeve);
757+
$this->rowSleeveMapper->update($rowSleeve);
756758
} catch (DoesNotExistException|MultipleObjectsReturnedException|Exception $e) {
757759
$this->logger->error($e->getMessage(), ['exception' => $e]);
758760
throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage());
@@ -761,9 +763,11 @@ public function update(Row2 $row): Row2 {
761763
$this->columnMapper->preloadColumns(array_column($changedCells, 'columnId'));
762764

763765
// write all changed cells to its db-table
766+
$cachedCells = $rowSleeve->getCachedCellsArray();
764767
foreach ($changedCells as $cell) {
765-
$this->insertOrUpdateCell($sleeve->getId(), $cell['columnId'], $cell['value']);
768+
$cachedCells[$cell['columnId']] = $this->insertOrUpdateCell($rowSleeve->getId(), $cell['columnId'], $cell['value']);
766769
}
770+
$rowSleeve->setCachedCells(json_encode($cachedCells));
767771

768772
return $row;
769773
}
@@ -815,9 +819,11 @@ private function updateMetaData($entity, bool $setCreate = false, ?string $lastE
815819
/**
816820
* Insert a cell to its specific db-table
817821
*
822+
* @return array<string, mixed> normalized cell data
823+
*
818824
* @throws InternalError
819825
*/
820-
private function insertCell(int $rowId, int $columnId, $value, ?string $lastEditAt = null, ?string $lastEditBy = null): void {
826+
private function insertCell(int $rowId, int $columnId, $value, ?string $lastEditAt = null, ?string $lastEditBy = null): array {
821827
try {
822828
$column = $this->columnMapper->find($columnId);
823829
} catch (DoesNotExistException $e) {
@@ -827,7 +833,7 @@ private function insertCell(int $rowId, int $columnId, $value, ?string $lastEdit
827833

828834
// insert new cell
829835
$cellMapper = $this->getCellMapper($column);
830-
836+
$cachedCell = [];
831837
try {
832838
$cellClassName = 'OCA\Tables\Db\RowCell' . ucfirst($column->getType());
833839
if ($cellMapper->hasMultipleValues()) {
@@ -839,6 +845,7 @@ private function insertCell(int $rowId, int $columnId, $value, ?string $lastEdit
839845
$this->updateMetaData($cell, false, $lastEditAt, $lastEditBy);
840846
$cellMapper->applyDataToEntity($column, $cell, $val);
841847
$cellMapper->insert($cell);
848+
$cachedCell[] = $cellMapper->toArray($cell);
842849
}
843850
} else {
844851
/** @var RowCellSuper $cell */
@@ -848,50 +855,62 @@ private function insertCell(int $rowId, int $columnId, $value, ?string $lastEdit
848855
$this->updateMetaData($cell, false, $lastEditAt, $lastEditBy);
849856
$cellMapper->applyDataToEntity($column, $cell, $value);
850857
$cellMapper->insert($cell);
858+
$cachedCell = $cellMapper->toArray($cell);
851859
}
852860
} catch (Exception $e) {
853861
$this->logger->error($e->getMessage(), ['exception' => $e]);
854862
throw new InternalError('Failed to insert column: ' . $e->getMessage(), 0, $e);
855863
}
864+
865+
return $cachedCell;
856866
}
857867

858868
/**
859869
* @param RowCellSuper $cell
860870
* @param RowCellMapperSuper $mapper
861871
* @param mixed $value the value should be parsed to the correct format within the row service
862872
* @param Column $column
873+
*
874+
* @return array<string, mixed> normalized cell data
863875
* @throws InternalError
864876
*/
865-
private function updateCell(RowCellSuper $cell, RowCellMapperSuper $mapper, $value, Column $column): void {
866-
$this->getCellMapper($column)->applyDataToEntity($column, $cell, $value);
877+
private function updateCell(RowCellSuper $cell, RowCellMapperSuper $mapper, $value, Column $column): array {
878+
$cellMapper = $this->getCellMapper($column);
879+
$cellMapper->applyDataToEntity($column, $cell, $value);
867880
$this->updateMetaData($cell);
868881
$mapper->updateWrapper($cell);
882+
883+
return $cellMapper->toArray($cell);
869884
}
870885

871886
/**
887+
* @return array<string, mixed> normalized cell data
872888
* @throws InternalError
873889
*/
874-
private function insertOrUpdateCell(int $rowId, int $columnId, $value): void {
890+
private function insertOrUpdateCell(int $rowId, int $columnId, $value): array {
875891
$column = $this->columnMapper->find($columnId);
876892
$cellMapper = $this->getCellMapper($column);
893+
$cachedCell = [];
877894
try {
878895
if ($cellMapper->hasMultipleValues()) {
879-
$this->atomic(function () use ($cellMapper, $rowId, $columnId, $value) {
896+
$this->atomic(function () use ($cellMapper, $rowId, $columnId, $value, &$cachedCell) {
880897
// For a usergroup field with mutiple values, each is inserted as a new cell
881898
// we need to delete all previous cells for this row and column, otherwise we get duplicates
882899
$cellMapper->deleteAllForColumnAndRow($columnId, $rowId);
883-
$this->insertCell($rowId, $columnId, $value);
900+
$cachedCell = $this->insertCell($rowId, $columnId, $value);
884901
}, $this->db);
885902
} else {
886903
$cell = $cellMapper->findByRowAndColumn($rowId, $columnId);
887-
$this->updateCell($cell, $cellMapper, $value, $column);
904+
$cachedCell = $this->updateCell($cell, $cellMapper, $value, $column);
888905
}
889906
} catch (DoesNotExistException) {
890-
$this->insertCell($rowId, $columnId, $value);
907+
$cachedCell = $this->insertCell($rowId, $columnId, $value);
891908
} catch (MultipleObjectsReturnedException|Exception $e) {
892909
$this->logger->error($e->getMessage(), ['exception' => $e]);
893-
throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage());
910+
throw new InternalError(get_class($this) . ' - ' . __FUNCTION__ . ': ' . $e->getMessage(), $e->getCode(), $e);
894911
}
912+
913+
return $cachedCell;
895914
}
896915

897916
private function getCellMapper(Column $column): RowCellMapperSuper {

‎lib/Db/RowCellMapperSuper.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@ public function applyDataToEntity(Column $column, RowCellSuper $cell, $data): vo
4949
$cell->setValue($data);
5050
}
5151

52+
public function toArray(RowCellSuper $cell): array {
53+
return ['value' => $cell->getValue()];
54+
}
55+
5256
public function getDbParamType() {
5357
return IQueryBuilder::PARAM_STR;
5458
}

‎lib/Db/RowCellUsergroupMapper.php‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,13 @@ public function formatRowData(Column $column, array $row) {
5858
];
5959
}
6060

61+
public function toArray(RowCellSuper $cell): array {
62+
return [
63+
'value' => $cell->getValue(),
64+
'value_type' => $cell->getValueType(),
65+
];
66+
}
67+
6168
public function hasMultipleValues(): bool {
6269
return true;
6370
}

‎lib/Db/RowSleeve.php‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
* @psalm-suppress PropertyNotSetInConstructor
1616
* @method getTableId(): ?int
1717
* @method setTableId(int $columnId)
18+
* @method getCachedCells(): string
19+
* @method setCachedCells(string $cachedCells)
1820
* @method getCreatedBy(): string
1921
* @method setCreatedBy(string $createdBy)
2022
* @method getCreatedAt(): string
@@ -26,6 +28,7 @@
2628
*/
2729
class RowSleeve extends Entity implements JsonSerializable {
2830
protected ?int $tableId = null;
31+
protected ?string $cachedCells = null;
2932
protected ?string $createdBy = null;
3033
protected ?string $createdAt = null;
3134
protected ?string $lastEditBy = null;
@@ -34,12 +37,21 @@ class RowSleeve extends Entity implements JsonSerializable {
3437
public function __construct() {
3538
$this->addType('id', 'integer');
3639
$this->addType('tableId', 'integer');
40+
$this->addType('cachedCells', 'string');
41+
}
42+
43+
/**
44+
* @return array<int, mixed> Indexed by column ID
45+
*/
46+
public function getCachedCellsArray(): array {
47+
return json_decode($this->cachedCells, true) ?: [];
3748
}
3849

3950
public function jsonSerialize(): array {
4051
return [
4152
'id' => $this->id,
4253
'tableId' => $this->tableId,
54+
'cachedCells' => $this->cachedCells,
4355
'createdBy' => $this->createdBy,
4456
'createdAt' => $this->createdAt,
4557
'lastEditBy' => $this->lastEditBy,

‎lib/Migration/CacheSleeveCells.php‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
<?php
2+
3+
/**
4+
* SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
5+
* SPDX-License-Identifier: AGPL-3.0-or-later
6+
*/
7+
8+
namespace OCA\Tables\Migration;
9+
10+
use OCA\Tables\Db\Row2Mapper;
11+
use OCP\IConfig;
12+
use OCP\IDBConnection;
13+
use OCP\Migration\IOutput;
14+
use OCP\Migration\IRepairStep;
15+
use Psr\Log\LoggerInterface;
16+
17+
class CacheSleeveCells implements IRepairStep {
18+
public function __construct(
19+
protected IDBConnection $db,
20+
protected Row2Mapper $rowMapper,
21+
protected IConfig $config,
22+
protected LoggerInterface $logger,
23+
) {
24+
}
25+
26+
/**
27+
* @inheritDoc
28+
*/
29+
public function getName() {
30+
return 'Caches cells to the row-sleeves table';
31+
}
32+
33+
/**
34+
* @inheritDoc
35+
*/
36+
public function run(IOutput $output) {
37+
$cachingSleeveCellsComplete = $this->config->getAppValue('tables', 'cachingSleeveCellsComplete', 'false') === 'true';
38+
if (!$cachingSleeveCellsComplete) {
39+
return;
40+
}
41+
42+
// fixme: get all tables
43+
// get columns for each tables
44+
// for each column call Row2Mapper::getRows
45+
while ($sleeveToProcess = $this->getSleeveToProcess()) {
46+
$rowIds = array_column($sleeveToProcess, 'id');
47+
}
48+
49+
$this->config->setAppValue('tables', 'cachingSleeveCellsComplete', 'true');
50+
}
51+
52+
private function getSleeveToProcess(): array
53+
{
54+
$qb = $this->db->getQueryBuilder();
55+
56+
return $qb->select('*')
57+
->from('tables_row_sleeves')
58+
->where($qb->expr()->isNull('cached_cells'))
59+
->setMaxResults(1000)
60+
->executeQuery()
61+
->fetchAll();
62+
}
63+
}

‎lib/Migration/Version001010Date20251229000000.php‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
<?php
2+
3+
/** @noinspection PhpUnused */
4+
5+
declare(strict_types=1);
6+
/**
7+
* SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
8+
* SPDX-License-Identifier: AGPL-3.0-or-later
9+
*/
10+
namespace OCA\Tables\Migration;
11+
12+
use Closure;
13+
use OCP\DB\Exception;
14+
use OCP\DB\ISchemaWrapper;
15+
use OCP\DB\Types;
16+
use OCP\Migration\IOutput;
17+
use OCP\Migration\SimpleMigrationStep;
18+
19+
class Version001010Date20251229000000 extends SimpleMigrationStep {
20+
/**
21+
* @param IOutput $output
22+
* @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
23+
* @param array $options
24+
* @return null|ISchemaWrapper
25+
* @throws Exception
26+
*/
27+
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
28+
/** @var ISchemaWrapper $schema */
29+
$schema = $schemaClosure();
30+
31+
$table = $schema->getTable('tables_row_sleeves');
32+
if (!$table->hasColumn('tables_row_sleeves')) {
33+
$table->addColumn('cells', Types::JSON, [
34+
'notnull' => false,
35+
]);
36+
}
37+
38+
return $schema;
39+
}
40+
}

0 commit comments

Comments
 (0)