Skip to content

Commit d4a0a7c

Browse files
abnegateabnegate
authored
Merge pull request #594 from utopia-php/fix-bulk-insert
Fix bulk inserts for sequece
2 parents 8e40af9 + 2e2a030 commit d4a0a7c

3 files changed

Lines changed: 98 additions & 24 deletions

File tree

‎src/Database/Adapter/MariaDB.php‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -954,16 +954,22 @@ public function createDocuments(string $collection, array $documents): array
954954
throw new DatabaseException('All documents must have an sequence if one is set');
955955
}
956956
}
957+
957958
$attributeKeys = array_unique($attributeKeys);
958959

960+
if ($hasSequence) {
961+
$attributeKeys[] = '_id';
962+
}
963+
959964
if ($this->sharedTables) {
960965
$attributeKeys[] = '_tenant';
961966
}
962967

963968
$columns = [];
964969
foreach ($attributeKeys as $key => $attribute) {
965-
$columns[$key] = "`{$this->filter($attribute)}`";
970+
$columns[$key] = $this->quote($this->filter($attribute));
966971
}
972+
967973
$columns = '(' . \implode(', ', $columns) . ')';
968974

969975
$bindIndex = 0;
@@ -982,7 +988,6 @@ public function createDocuments(string $collection, array $documents): array
982988

983989
if (!empty($document->getSequence())) {
984990
$attributes['_id'] = $document->getSequence();
985-
$attributeKeys[] = '_id';
986991
} else {
987992
$documentIds[] = $document->getId();
988993
}
@@ -1007,6 +1012,7 @@ public function createDocuments(string $collection, array $documents): array
10071012
}
10081013

10091014
$batchKeys[] = '(' . \implode(', ', $bindKeys) . ')';
1015+
10101016
foreach (Database::PERMISSIONS as $type) {
10111017
foreach ($document->getPermissionsByType($type) as $permission) {
10121018
$tenantBind = $this->sharedTables ? ", :_tenant_{$index}" : '';

‎src/Database/Adapter/Postgres.php‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1051,6 +1051,7 @@ public function createDocument(string $collection, Document $document): Document
10511051
* @return array<Document>
10521052
*
10531053
* @throws DuplicateException
1054+
* @throws \Throwable
10541055
*/
10551056
public function createDocuments(string $collection, array $documents): array
10561057
{
@@ -1060,37 +1061,44 @@ public function createDocuments(string $collection, array $documents): array
10601061

10611062
try {
10621063
$name = $this->filter($collection);
1064+
10631065
$attributeKeys = Database::INTERNAL_ATTRIBUTE_KEYS;
10641066

10651067
$hasSequence = null;
10661068
foreach ($documents as $document) {
10671069
$attributes = $document->getAttributes();
1068-
$attributeKeys = array_merge($attributeKeys, array_keys($attributes));
1070+
$attributeKeys = [...$attributeKeys, ...\array_keys($attributes)];
10691071

10701072
if ($hasSequence === null) {
10711073
$hasSequence = !empty($document->getSequence());
10721074
} elseif ($hasSequence == empty($document->getSequence())) {
10731075
throw new DatabaseException('All documents must have an sequence if one is set');
10741076
}
10751077
}
1078+
10761079
$attributeKeys = array_unique($attributeKeys);
10771080

1081+
if ($hasSequence) {
1082+
$attributeKeys[] = '_id';
1083+
}
1084+
10781085
if ($this->sharedTables) {
10791086
$attributeKeys[] = '_tenant';
10801087
}
10811088

10821089
$columns = [];
10831090
foreach ($attributeKeys as $key => $attribute) {
1084-
$columns[$key] = "\"{$this->filter($attribute)}\"";
1091+
$columns[$key] = $this->quote($this->filter($attribute));
10851092
}
1086-
$columns = '(' . \implode(', ', $columns) . ')';
10871093

1088-
$sequences = [];
1094+
$columns = '(' . \implode(', ', $columns) . ')';
10891095

10901096
$bindIndex = 0;
10911097
$batchKeys = [];
10921098
$bindValues = [];
10931099
$permissions = [];
1100+
$documentIds = [];
1101+
$documentTenants = [];
10941102

10951103
foreach ($documents as $index => $document) {
10961104
$attributes = $document->getAttributes();
@@ -1100,13 +1108,14 @@ public function createDocuments(string $collection, array $documents): array
11001108
$attributes['_permissions'] = \json_encode($document->getPermissions());
11011109

11021110
if (!empty($document->getSequence())) {
1103-
$sequences[$document->getId()] = true;
11041111
$attributes['_id'] = $document->getSequence();
1105-
$attributeKeys[] = '_id';
1112+
} else {
1113+
$documentIds[] = $document->getId();
11061114
}
11071115

11081116
if ($this->sharedTables) {
11091117
$attributes['_tenant'] = $document->getTenant();
1118+
$documentTenants[] = $document->getTenant();
11101119
}
11111120

11121121
$bindKeys = [];
@@ -1124,6 +1133,7 @@ public function createDocuments(string $collection, array $documents): array
11241133
}
11251134

11261135
$batchKeys[] = '(' . \implode(', ', $bindKeys) . ')';
1136+
11271137
foreach (Database::PERMISSIONS as $type) {
11281138
foreach ($document->getPermissionsByType($type) as $permission) {
11291139
$tenantBind = $this->sharedTables ? ", :_tenant_{$index}" : '';
@@ -1167,18 +1177,20 @@ public function createDocuments(string $collection, array $documents): array
11671177

11681178
$this->execute($stmtPermissions);
11691179
}
1170-
} catch (PDOException $e) {
1171-
throw $this->processException($e);
1172-
}
11731180

1174-
foreach ($documents as $document) {
1175-
if (!isset($sequences[$document->getId()])) {
1176-
$document['$sequence'] = $this->getDocument(
1177-
$collection,
1178-
$document->getId(),
1179-
[Query::select(['$sequence'])]
1180-
)->getSequence();
1181+
$sequences = $this->getSequences(
1182+
$collection,
1183+
$documentIds,
1184+
$documentTenants
1185+
);
1186+
1187+
foreach ($documents as $document) {
1188+
if (isset($sequences[$document->getId()])) {
1189+
$document['$sequence'] = $sequences[$document->getId()];
1190+
}
11811191
}
1192+
} catch (PDOException $e) {
1193+
throw $this->processException($e);
11821194
}
11831195

11841196
return $documents;

‎tests/e2e/Adapter/Scopes/DocumentTests.php‎

Lines changed: 62 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -208,10 +208,8 @@ public function testCreateDocument(): Document
208208

209209
return $document;
210210
}
211-
/**
212-
* @return array<Document>
213-
*/
214-
public function testCreateDocuments(): array
211+
212+
public function testCreateDocuments(): void
215213
{
216214
$count = 3;
217215
$collection = 'testCreateDocuments';
@@ -242,7 +240,27 @@ public function testCreateDocuments(): array
242240
]);
243241
}
244242

245-
$count = $database->createDocuments($collection, $documents, 3);
243+
$results = [];
244+
245+
$count = $database->createDocuments($collection, $documents, 3, onNext: function ($doc) use (&$results) {
246+
$results[] = $doc;
247+
});
248+
249+
$this->assertEquals($count, \count($results));
250+
251+
foreach ($results as $document) {
252+
$this->assertNotEmpty(true, $document->getId());
253+
$this->assertIsString($document->getAttribute('string'));
254+
$this->assertEquals('text📝', $document->getAttribute('string')); // Also makes sure an emoji is working
255+
$this->assertIsInt($document->getAttribute('integer'));
256+
$this->assertEquals(5, $document->getAttribute('integer'));
257+
$this->assertIsInt($document->getAttribute('bigint'));
258+
$this->assertEquals(9223372036854775807, $document->getAttribute('bigint'));
259+
}
260+
261+
$documents = $database->find($collection, [
262+
Query::orderAsc()
263+
]);
246264

247265
$this->assertEquals($count, \count($documents));
248266

@@ -255,8 +273,46 @@ public function testCreateDocuments(): array
255273
$this->assertIsInt($document->getAttribute('bigint'));
256274
$this->assertEquals(9223372036854775807, $document->getAttribute('bigint'));
257275
}
276+
}
258277

259-
return $documents;
278+
public function testCreateDocumentsWithAutoIncrement(): void
279+
{
280+
/** @var Database $database */
281+
$database = static::getDatabase();
282+
283+
$database->createCollection(__FUNCTION__);
284+
285+
$this->assertEquals(true, $database->createAttribute(__FUNCTION__, 'string', Database::VAR_STRING, 128, true));
286+
287+
/** @var array<Document> $documents */
288+
$documents = [];
289+
$count = 10;
290+
$sequence = 1_000_000;
291+
292+
for ($i = $sequence; $i <= ($sequence + $count); $i++) {
293+
$documents[] = new Document([
294+
'$sequence' => (string)$i,
295+
'$permissions' => [
296+
Permission::read(Role::any()),
297+
Permission::create(Role::any()),
298+
Permission::update(Role::any()),
299+
Permission::delete(Role::any()),
300+
],
301+
'string' => 'text',
302+
]);
303+
}
304+
305+
$count = $database->createDocuments(__FUNCTION__, $documents, 6);
306+
$this->assertEquals($count, \count($documents));
307+
308+
$documents = $database->find(__FUNCTION__, [
309+
Query::orderAsc()
310+
]);
311+
foreach ($documents as $index => $document) {
312+
$this->assertEquals($sequence + $index, $document->getSequence());
313+
$this->assertNotEmpty(true, $document->getId());
314+
$this->assertEquals('text', $document->getAttribute('string'));
315+
}
260316
}
261317

262318
public function testCreateDocumentsWithDifferentAttributes(): void

0 commit comments

Comments
 (0)