From c651c220a7520f81089a3b6a2bef22bb49304a33 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 14 Jul 2025 11:50:06 +0300 Subject: [PATCH 01/16] transactions support --- bin/init-mongo-replica-set.sh | 32 +++++ composer.json | 2 +- composer.lock | 28 ++-- docker-compose.yml | 13 +- mongo-keyfile | 16 +++ src/Database/Adapter.php | 3 - src/Database/Adapter/MariaDB.php | 1 - src/Database/Adapter/Mongo.php | 225 +++++++++++++++++++++++------- tests/e2e/Adapter/MongoDBTest.php | 35 +++++ 9 files changed, 288 insertions(+), 67 deletions(-) create mode 100644 bin/init-mongo-replica-set.sh create mode 100644 mongo-keyfile diff --git a/bin/init-mongo-replica-set.sh b/bin/init-mongo-replica-set.sh new file mode 100644 index 000000000..4233e4545 --- /dev/null +++ b/bin/init-mongo-replica-set.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +set -e + +echo "Waiting for MongoDB to be ready..." +until docker compose exec mongo mongosh --eval "print('MongoDB is ready')" > /dev/null 2>&1; do + sleep 1 +done + +echo "Initializing MongoDB replica set..." + +# First, initialize the replica set without authentication +echo "Initializing replica set..." +docker compose exec mongo mongosh --eval 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongo:27017"}]})' + +# Wait for the replica set to be ready +echo "Waiting for replica set to be ready..." +until docker compose exec mongo mongosh --eval "rs.status().ok" | grep -q "1"; do + sleep 2 +done + +echo "Replica set initialized successfully!" + +# Now create the admin user and enable authentication +echo "Creating admin user and enabling authentication..." +docker compose exec mongo mongosh --eval 'use admin; db.createUser({user: "root", pwd: "password", roles: [{role: "root", db: "admin"}]})' + +# Test authentication +echo "Testing authentication..." +docker compose exec mongo mongosh admin -u root -p password --eval 'db.runCommand({ping: 1})' + +echo "MongoDB replica set is ready for transactions!" \ No newline at end of file diff --git a/composer.json b/composer.json index 7300239c4..c03a6747d 100755 --- a/composer.json +++ b/composer.json @@ -39,7 +39,7 @@ "utopia-php/framework": "0.33.*", "utopia-php/cache": "0.13.*", "utopia-php/pools": "0.8.*", - "utopia-php/mongo": "dev-feat-bulk-writes as 0.3.1" + "utopia-php/mongo": "dev-feat-mongo-transactions as 0.3.1" }, "require-dev": { "fakerphp/faker": "1.23.*", diff --git a/composer.lock b/composer.lock index 59db2f26b..2bc7a510d 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cd1babfd7f7750ad399c915edd6209ad", + "content-hash": "a3b0ff08e6addea30a6380a0b19a0529", "packages": [ { "name": "brick/math", @@ -1993,16 +1993,16 @@ }, { "name": "utopia-php/mongo", - "version": "dev-feat-bulk-writes", + "version": "dev-feat-mongo-transactions", "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "414d4d099386ba742d1620fe2a75afd6105ad611" + "reference": "71c062c42bca6a4f709ddb86670089d5d4a5d7d5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/414d4d099386ba742d1620fe2a75afd6105ad611", - "reference": "414d4d099386ba742d1620fe2a75afd6105ad611", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/71c062c42bca6a4f709ddb86670089d5d4a5d7d5", + "reference": "71c062c42bca6a4f709ddb86670089d5d4a5d7d5", "shasum": "" }, "require": { @@ -2047,9 +2047,9 @@ ], "support": { "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/feat-bulk-writes" + "source": "https://github.com/utopia-php/mongo/tree/feat-mongo-transactions" }, - "time": "2025-07-08T17:47:22+00:00" + "time": "2025-07-14T08:20:00+00:00" }, { "name": "utopia-php/pools", @@ -2290,16 +2290,16 @@ }, { "name": "laravel/pint", - "version": "v1.23.0", + "version": "v1.24.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "9ab851dba4faa51a3c3223dd3d07044129021024" + "reference": "0345f3b05f136801af8c339f9d16ef29e6b4df8a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/9ab851dba4faa51a3c3223dd3d07044129021024", - "reference": "9ab851dba4faa51a3c3223dd3d07044129021024", + "url": "https://api.github.com/repos/laravel/pint/zipball/0345f3b05f136801af8c339f9d16ef29e6b4df8a", + "reference": "0345f3b05f136801af8c339f9d16ef29e6b4df8a", "shasum": "" }, "require": { @@ -2310,7 +2310,7 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.76.0", + "friendsofphp/php-cs-fixer": "^3.82.2", "illuminate/view": "^11.45.1", "larastan/larastan": "^3.5.0", "laravel-zero/framework": "^11.45.0", @@ -2355,7 +2355,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-07-03T10:37:47+00:00" + "time": "2025-07-10T18:09:32+00:00" }, { "name": "myclabs/deep-copy", @@ -4261,7 +4261,7 @@ "aliases": [ { "package": "utopia-php/mongo", - "version": "dev-feat-bulk-writes", + "version": "dev-feat-mongo-transactions", "alias": "0.3.1", "alias_normalized": "0.3.1.0" } diff --git a/docker-compose.yml b/docker-compose.yml index 1af22637e..26911a3af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,10 +79,19 @@ services: - database ports: - "9706:27017" + volumes: + - ./mongo-keyfile:/etc/mongo-keyfile:ro + - mongo-data:/data/db environment: MONGO_INITDB_DATABASE: utopia_testing MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: password + MONGO_INITDB_ROOT_PASSWORD: password + command: > + mongod --replSet rs0 + --auth + --keyFile /etc/mongo-keyfile +# Manyally initate the replica set +#docker compose exec mongo mongosh admin -u root -p password --eval 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongo:27017"}]})' mongo-express: image: mongo-express @@ -146,5 +155,7 @@ services: networks: - database +volumes: + mongo-data: networks: database: diff --git a/mongo-keyfile b/mongo-keyfile new file mode 100644 index 000000000..5585939eb --- /dev/null +++ b/mongo-keyfile @@ -0,0 +1,16 @@ +ydIuYSvU/9QLt7fkH32IdXbP2z2+w+fzSEoolW8Q1Z8nLhRyrZF0Zq7a0KzeNI7K +gPIl1ikI6ob6h0+RxYmGeOOUjjkcBlkvYrmABDKsRipTkTTp4z0fUBTIUJV0lVvs +N9+VpM0/pLLIhI8jb38aa7pmsoufBQ3uiNR68ZFykPqzZQ4d5VfMqfZk7z3dpFlh +DURPOOG0HAFe68MLXVFYdaHGW4yomuTPrpzWSiUhFAPFEBYg4elARQc4CaiinFds +SQi/SrUsYMGODPr+on9/lboia/SInaSP+dzDqpsbL29atvIVHtU29RlPJdZ2V1ub +Oe2O1xN9F59TtjNUgDiAtMGKTMS/0S1mbPC6Og5JAR7U4xZ7/6S5n3+p0RjYyTlH +fhssJ7pc/bveN6mShNrsIKK0Z50YYjablzm07EDJYhfEWMG5Wu1AvEVqEH68ioDl +JL5QO63A2bXvMN7dXS69+E0hHn6xaZYu+CnKedvgWdyhraCT1Q01ZyDyv2y7isGD +1BAlNLlt+cPMCitETcxZne+JHdkL/mDKffHUPM4Drtzchg4DbiG49uC9Ib7zTws+ +NcburXY+9B8j7WN7ZHXhiB7/OWJ/IHJCZTdKz70mEPH4AHoRFpZNM5eMnYxYdbQD +40MhAS7fuOYhtFIQiQ+SCeFMucE3KYvp1JpTVQwT4SNrIlHPqfPn5xFBcgDjhvwT +hHJCgXP4HrRuf47Ta6kHy2UFQ7r5JOqSZSOFwP+tUyfhjEB5ZWJ1qCUZxFagoc9A +//9SoyulZwCxEr2ijmes1Nzv56hSTjYb6pPjFWd92G87w+VZv4R/vF5nwcYUyuIS +iQWPs/kOzb4NeJW24lNzR2zH2BsJt3OI+BFY64cc8O0o6EtFWcoabwyJYKe6RXPX +0S4ngcnGzRP+tVa6LsrjAYrNpmZDrP9x93pXQHfByTS2oSaI1eGeAagFTu/HS2kC +uCJ0HfH99sRSgJ1Ab+2C8G8305meDAbtdCtvl/1anPnV6ISy diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 3d59e3744..490d058cf 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -374,10 +374,7 @@ public function withTransaction(callable $callback): mixed for ($attempts = 0; $attempts < 3; $attempts++) { try { $this->startTransaction(); - //var_dump($attempts); $result = $callback(); - //var_dump($result); - $this->commitTransaction(); return $result; diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 8062839c8..dc626b665 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1173,7 +1173,6 @@ public function createOrUpdateDocuments( $bindValues = []; $documentIds = []; $documentTenants = []; - foreach ($changes as $change) { $document = $change->getNew(); $attributes = $document->getAttributes(); diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 5fade391a..6836b9422 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -11,7 +11,6 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception as DatabaseException; -use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Duplicate; use Utopia\Database\Exception\Timeout; use Utopia\Database\Query; @@ -44,6 +43,14 @@ class Mongo extends Adapter //protected ?int $timeout = null; + /** + * Transaction/session state for MongoDB transactions + */ + private ?object $sessionId = null; // Store raw BSON id object + private ?int $txnNumber = null; + protected int $inTransaction = 0; + private bool $firstOpInTransaction = false; + /** * Constructor. * @@ -74,19 +81,153 @@ public function clearTimeout(string $event): void $this->timeout = 0; } + /** + * @template T + * @param callable(): T $callback + * @return T + * @throws \Throwable + */ + public function withTransaction(callable $callback): mixed + { + // We removed the attmpts to retry the transaction. + // Since if it's rolling back the second time, it will fail + //becouse we already run one abortTransaction. + try { + $this->startTransaction(); + $result = $callback(); + $this->commitTransaction(); + return $result; + } catch (\Throwable $action) { + try { + $this->rollbackTransaction(); + } catch (\Throwable $rollback) { + $this->inTransaction = 0; + // Throw the original exception, not the rollback one + // Since if it's a duplicate key error, the rollback will fail + //and we want to throw the original exception. + } + $this->inTransaction = 0; + throw $action; + } + } + + public function startTransaction(): bool { - return true; + try { + if ($this->inTransaction === 0) { + if (!$this->sessionId) { + $this->sessionId = $this->client->startSession(); // Store raw id object + } + $this->txnNumber = ($this->txnNumber ?? 0) + 1; + $this->firstOpInTransaction = true; + + // Initialize the transaction on MongoDB's side with a dummy find operation + // This ensures the transaction is active even if validation fails later. + $this->client->query([ + 'find' => 'system.version', + 'filter' => $this->client->toObject([]), + 'limit' => 1, + 'lsid' => ['id' => $this->sessionId], + 'txnNumber' => new \MongoDB\BSON\Int64($this->txnNumber), // Long type for txnNumber + 'autocommit' => false, + 'startTransaction' => true + ], 'admin'); + + $this->firstOpInTransaction = false; + } + $this->inTransaction++; + return true; + } catch (\Throwable $e) { + throw new DatabaseException('Failed to start transaction: ' . $e->getMessage(), $e->getCode(), $e); + } } public function commitTransaction(): bool { - return true; + try { + if ($this->inTransaction === 0) { + throw new DatabaseException('No transaction in progress'); + } + $this->inTransaction--; + if ($this->inTransaction === 0) { + if (!$this->sessionId) { + throw new DatabaseException('No session in progress'); + } + $result = $this->client->commitTransaction( + ['id' => $this->sessionId], // Pass raw id object + $this->txnNumber, + false + ); + if (($result->ok ?? 0) !== 1.0) { + throw new DatabaseException('Failed to commit transaction'); + } + + // Session is now closed by the client using endSessions, reset our state + $this->sessionId = null; + $this->txnNumber = null; + + return true; + } + return true; + } catch (\Throwable $e) { + throw new DatabaseException('Failed to commit transaction: ' . $e->getMessage(), $e->getCode(), $e); + } } public function rollbackTransaction(): bool { - return true; + + try { + if ($this->inTransaction === 0) { + throw new DatabaseException('No transaction in progress'); + } + $this->inTransaction--; + if ($this->inTransaction === 0) { + if (!$this->sessionId) { + throw new DatabaseException('No session in progress'); + } + + $result = $this->client->abortTransaction( + ['id' => $this->sessionId], // Pass raw id object + $this->txnNumber, + false + ); + if (($result->ok ?? 0) !== 1.0) { + throw new DatabaseException('Failed to rollback transaction'); + } + + // Session is now closed by the client using endSessions, reset our state + $this->sessionId = null; + $this->txnNumber = null; + + return true; + } + return true; + } catch (\Throwable $e) { + throw new DatabaseException('Failed to rollback transaction: ' . $e->getMessage(), $e->getCode(), $e); + } + } + + /** + * Helper to add transaction/session context to command options if in transaction + */ + private function addTransactionContext(array $options = []): array + { + + if ($this->inTransaction) { + $options['lsid'] = ['id' => $this->sessionId]; + $options['txnNumber'] = new \MongoDB\BSON\Int64($this->txnNumber); + $options['autocommit'] = false; + + if ($this->firstOpInTransaction) { + // For MongoDB, the first operation in a transaction should include startTransaction + $options['startTransaction'] = true; + $this->firstOpInTransaction = false; + } + + } + return $options; } /** @@ -199,7 +340,6 @@ public function createCollection(string $name, array $attributes = [], array $in // Returns an array/object with the result document try { $this->getClient()->createCollection($id); - } catch (MongoException $e) { throw new Duplicate($e->getMessage(), $e->getCode(), $e); } @@ -394,7 +534,6 @@ public function createAttributes(string $collection, array $attributes): bool public function deleteAttribute(string $collection, string $id): bool { $collection = $this->getNamespace() . '_' . $this->filter($collection); - $this->getClient()->update( $collection, [], @@ -595,7 +734,6 @@ public function createIndex(string $collection, string $id, string $type, array $id = $this->filter($id); $indexes = []; - $options = []; // pass in custom index name $indexes['name'] = $id; @@ -636,7 +774,7 @@ public function createIndex(string $collection, string $id, string $type, array ]; } - return $this->client->createIndexes($name, [$indexes], $options); + return $this->client->createIndexes($name, [$indexes]); } /** @@ -767,9 +905,8 @@ public function createDocument(string $collection, Document $document): Document if (!empty($sequence)) { $record['_id'] = $sequence; } - - $result = $this->insertDocument($name, $this->removeNullKeys($record)); - + $options = $this->addTransactionContext([]); + $result = $this->insertDocument($name, $this->removeNullKeys($record), $options); $result = $this->replaceChars('_', '$', $result); $result = $this->timeToDocument($result); @@ -790,6 +927,9 @@ public function createDocuments(string $collection, array $documents): array { $name = $this->getNamespace() . '_' . $this->filter($collection); + // Initialize transaction context before validation to ensure transaction is active + $options = $this->addTransactionContext([]); + $records = []; $hasSequence = null; $documents = array_map(fn ($doc) => clone $doc, $documents); @@ -819,7 +959,7 @@ public function createDocuments(string $collection, array $documents): array $records[] = $this->removeNullKeys($record); } - $documents = $this->client->insertMany($name, $records); + $documents = $this->client->insertMany($name, $records, $options); foreach ($documents as $index => $document) { $documents[$index] = $this->replaceChars('_', '$', $this->client->toArray($document)); @@ -839,26 +979,12 @@ public function createDocuments(string $collection, array $documents): array * @return array * @throws Duplicate */ - private function insertDocument(string $name, array $document): array + private function insertDocument(string $name, array $document, array $options = []): array { try { - $bla = $this->client->insert($name, $document); - - $filters = []; - $filters['_uid'] = $document['_uid']; - - if ($this->sharedTables) { - $filters['_tenant'] = $this->getTenant(); - } - - $result = $this->client->find( - $name, - $filters, - ['limit' => 1] - )->cursor->firstBatch[0]; - - return $this->client->toArray($result); + $result = $this->client->insert($name, $document, $options); + return $result; } catch (MongoException $e) { throw new Duplicate($e->getMessage()); } @@ -890,7 +1016,8 @@ public function updateDocument(string $collection, string $id, Document $documen try { unset($record['_id']); // Don't update _id - $this->client->update($name, $filters, $record); + $options = $this->addTransactionContext([]); + $this->client->update($name, $filters, $record, $options); } catch (MongoException $e) { throw new Duplicate($e->getMessage()); } @@ -934,7 +1061,8 @@ public function updateDocuments(string $collection, Document $updates, array $do ]; try { - $this->client->update($name, $filters, $updateQuery, multi: true); + $options = $this->addTransactionContext([]); + $this->client->update($name, $filters, $updateQuery, multi: true, options: $options); } catch (MongoException $e) { throw new Duplicate($e->getMessage()); } @@ -1009,13 +1137,13 @@ public function createOrUpdateDocuments(string $collection, string $attribute, a 'filter' => $filter, 'update' => $update, ]; - } - - // Use the new bulkUpsert method + } + + $options = $this->addTransactionContext([]); $this->client->bulkUpsert( $name, $operations, - ["ordered" => false] // TODO Do we want to continue if an error is thrown? + options: $options ); // Get sequences for documents that were created @@ -1106,6 +1234,7 @@ public function increaseDocumentAttribute(string $collection, string $id, string $filters[$attribute] = ['$gte' => $min]; } + $options = $this->addTransactionContext([]); $this->client->update( $this->getNamespace() . '_' . $this->filter($collection), $filters, @@ -1113,6 +1242,7 @@ public function increaseDocumentAttribute(string $collection, string $id, string '$inc' => [$attribute => $value], '$set' => ['_updatedAt' => $this->toMongoDatetime($updatedAt)], ], + options: $options ); return true; @@ -1137,7 +1267,8 @@ public function deleteDocument(string $collection, string $id): bool $filters['_tenant'] = $this->getTenant(); } - $result = $this->client->delete($name, $filters); + $options = $this->addTransactionContext([]); + $result = $this->client->delete($name, $filters, 1, [], $options); return (!!$result); } @@ -1162,15 +1293,15 @@ public function deleteDocuments(string $collection, array $sequences, array $per $filters = $this->replaceInternalIdsKeys($filters, '$', '_', $this->operators); $filters = $this->timeFilter($filters); - - $options = []; + $options = $this->addTransactionContext([]); try { $count = $this->client->delete( collection: $name, filters: $filters, - options: $options, - limit: 0 + limit: 0, + deleteOptions: [], + options: $options ); } catch (MongoException $e) { $this->processException($e); @@ -1249,7 +1380,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, if ($this->sharedTables) { $filters['_tenant'] = $this->getTenant(); } - + // permissions if (Authorization::$status) { $roles = \implode('|', Authorization::getRoles()); @@ -1306,7 +1437,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $prevAttr = $this->filter($this->getInternalKeyForAttribute($originalPrev)); $tmp = $cursor[$originalPrev]; - if($originalPrev === '$sequence'){ + if ($originalPrev === '$sequence') { $tmp = new ObjectId($tmp); } @@ -1317,11 +1448,11 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $tmp = $cursor[$originalAttribute]; - if($originalAttribute === '$sequence'){ + if ($originalAttribute === '$sequence') { $tmp = new ObjectId($tmp); /** If there is only $sequence attribute in $orderAttributes skip Or And operators **/ - if(count($orderAttributes) === 1){ + if (count($orderAttributes) === 1) { $filters[$attribute] = [ $operator => $tmp ]; @@ -1356,16 +1487,16 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, } catch (MongoException $e) { throw $this->processException($e); } - + if (empty($results)) { return $found; } - + foreach ($this->client->toArray($results) as $result) { $record = $this->replaceChars('_', '$', (array)$result); - + $record = $this->timeToDocument($record); $found[] = new Document($record); diff --git a/tests/e2e/Adapter/MongoDBTest.php b/tests/e2e/Adapter/MongoDBTest.php index 55b21f8e4..2e18f8058 100644 --- a/tests/e2e/Adapter/MongoDBTest.php +++ b/tests/e2e/Adapter/MongoDBTest.php @@ -105,4 +105,39 @@ protected static function deleteIndex(string $collection, string $index): bool { return true; } + + /** + * Test that sessions are properly closed after transactions + */ + public function testSessionCleanup(): void + { + $database = static::getDatabase(); + $adapter = $database->getAdapter(); + + // Create a collection for testing + $collection = $database->createCollection('test_session_cleanup'); + + // Start a transaction + $adapter->startTransaction(); + + // Create a document in the transaction + $document = $database->createDocument('test_session_cleanup', new \Utopia\Database\Document([ + 'name' => 'test', + 'value' => 123 + ])); + + // Commit the transaction - session is closed using endSessions command + $adapter->commitTransaction(); + + // Verify the document was created + $this->assertNotNull($document->getId()); + + // The session should now be closed (sessionId should be null) + // We can verify this by checking that a new transaction starts fresh + $adapter->startTransaction(); + $adapter->rollbackTransaction(); + + // Clean up + $database->deleteCollection('test_session_cleanup'); + } } From b262c2168a79705b9fef8b0dadeb55696fa0f576 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 14 Jul 2025 18:48:53 +0300 Subject: [PATCH 02/16] remove replica-set file --- bin/init-mongo-replica-set.sh | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 bin/init-mongo-replica-set.sh diff --git a/bin/init-mongo-replica-set.sh b/bin/init-mongo-replica-set.sh deleted file mode 100644 index 4233e4545..000000000 --- a/bin/init-mongo-replica-set.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -set -e - -echo "Waiting for MongoDB to be ready..." -until docker compose exec mongo mongosh --eval "print('MongoDB is ready')" > /dev/null 2>&1; do - sleep 1 -done - -echo "Initializing MongoDB replica set..." - -# First, initialize the replica set without authentication -echo "Initializing replica set..." -docker compose exec mongo mongosh --eval 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongo:27017"}]})' - -# Wait for the replica set to be ready -echo "Waiting for replica set to be ready..." -until docker compose exec mongo mongosh --eval "rs.status().ok" | grep -q "1"; do - sleep 2 -done - -echo "Replica set initialized successfully!" - -# Now create the admin user and enable authentication -echo "Creating admin user and enabling authentication..." -docker compose exec mongo mongosh --eval 'use admin; db.createUser({user: "root", pwd: "password", roles: [{role: "root", db: "admin"}]})' - -# Test authentication -echo "Testing authentication..." -docker compose exec mongo mongosh admin -u root -p password --eval 'db.runCommand({ping: 1})' - -echo "MongoDB replica set is ready for transactions!" \ No newline at end of file From ba11cba078c70343b105a4df380b3c1cd1843ea7 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 15 Jul 2025 15:10:27 +0300 Subject: [PATCH 03/16] cleanup --- src/Database/Adapter/Mongo.php | 40 ++++++++++++++++++++++++------- tests/e2e/Adapter/MongoDBTest.php | 38 +++-------------------------- 2 files changed, 34 insertions(+), 44 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 6836b9422..a0f110174 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -89,9 +89,10 @@ public function clearTimeout(string $event): void */ public function withTransaction(callable $callback): mixed { - // We removed the attmpts to retry the transaction. - // Since if it's rolling back the second time, it will fail - //becouse we already run one abortTransaction. + // Removed the attmpts to retry the transaction. + //Unlike pdo if we run theabortTransaction more then once (same transactioId), + // it will throw an error the there is no transaction in progress. + try { $this->startTransaction(); $result = $callback(); @@ -163,7 +164,8 @@ public function commitTransaction(): bool throw new DatabaseException('Failed to commit transaction'); } - // Session is now closed by the client using endSessions, reset our state + // Session is now closed by the client using endSessions, state is reseted + // TODO do we want session per transaction or to manage it on the connection level? $this->sessionId = null; $this->txnNumber = null; @@ -225,7 +227,6 @@ private function addTransactionContext(array $options = []): array $options['startTransaction'] = true; $this->firstOpInTransaction = false; } - } return $options; } @@ -865,14 +866,14 @@ public function getDocument(string $collection, string $id, array $queries = [], } $result = $this->client->find($name, $filters, $options)->cursor->firstBatch; - + if (empty($result)) { return new Document([]); } $result = $this->replaceChars('_', '$', (array)$result[0]); $result = $this->timeToDocument($result); - + return new Document($result); } @@ -981,10 +982,31 @@ public function createDocuments(string $collection, array $documents): array */ private function insertDocument(string $name, array $document, array $options = []): array { - + try { $result = $this->client->insert($name, $document, $options); - return $result; + + + $filters = []; + $filters['_uid'] = $document['_uid']; + + if ($this->sharedTables) { + $filters['_tenant'] = $this->getTenant(); + } + + // in order to get the document we need to pass the transaction context to the find. + $this->client->find( + $name, + $filters, + array_merge($options, ['limit' => 1]) + )->cursor->firstBatch[0]; + + /** + * TODO Do we even need this find? + * We can just return the result from the insertDocument. + */ + + return $this->client->toArray($result); } catch (MongoException $e) { throw new Duplicate($e->getMessage()); } diff --git a/tests/e2e/Adapter/MongoDBTest.php b/tests/e2e/Adapter/MongoDBTest.php index 2e18f8058..4f16b9581 100644 --- a/tests/e2e/Adapter/MongoDBTest.php +++ b/tests/e2e/Adapter/MongoDBTest.php @@ -9,6 +9,9 @@ use Utopia\Database\Adapter\Mongo; use Utopia\Database\Database; use Utopia\Mongo\Client; +use Utopia\Database\Helpers\Permission; +use Utopia\Database\Helpers\Role; +use Utopia\Database\Document; class MongoDBTest extends Base { @@ -105,39 +108,4 @@ protected static function deleteIndex(string $collection, string $index): bool { return true; } - - /** - * Test that sessions are properly closed after transactions - */ - public function testSessionCleanup(): void - { - $database = static::getDatabase(); - $adapter = $database->getAdapter(); - - // Create a collection for testing - $collection = $database->createCollection('test_session_cleanup'); - - // Start a transaction - $adapter->startTransaction(); - - // Create a document in the transaction - $document = $database->createDocument('test_session_cleanup', new \Utopia\Database\Document([ - 'name' => 'test', - 'value' => 123 - ])); - - // Commit the transaction - session is closed using endSessions command - $adapter->commitTransaction(); - - // Verify the document was created - $this->assertNotNull($document->getId()); - - // The session should now be closed (sessionId should be null) - // We can verify this by checking that a new transaction starts fresh - $adapter->startTransaction(); - $adapter->rollbackTransaction(); - - // Clean up - $database->deleteCollection('test_session_cleanup'); - } } From 72c903e557e6b1ca02a72363622765fab34f4dfc Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 21 Jul 2025 10:13:03 +0300 Subject: [PATCH 04/16] clenup --- src/Database/Adapter/Mongo.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index d0468bae4..62feb2323 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -341,7 +341,6 @@ public function createCollection(string $name, array $attributes = [], array $in // Returns an array/object with the result document try { $this->getClient()->createCollection($id); - } catch (MongoException $e) { throw new Duplicate($e->getMessage(), $e->getCode(), $e); } @@ -536,7 +535,6 @@ public function createAttributes(string $collection, array $attributes): bool public function deleteAttribute(string $collection, string $id): bool { $collection = $this->getNamespace() . '_' . $this->filter($collection); - $this->getClient()->update( $collection, [], @@ -869,6 +867,7 @@ public function getDocument(string $collection, string $id, array $queries = [], } $result = $this->client->find($name, $filters, $options)->cursor->firstBatch; + if (empty($result)) { return new Document([]); } From 8421a343521e7aa1d2f459f28da674e4d29c62d4 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 21 Jul 2025 13:19:52 +0300 Subject: [PATCH 05/16] sync with feat-mongo-2 --- composer.lock | 113 +++++++++++++++++++++++++----- src/Database/Adapter/Mongo.php | 25 +++---- tests/e2e/Adapter/MongoDBTest.php | 3 - 3 files changed, 108 insertions(+), 33 deletions(-) diff --git a/composer.lock b/composer.lock index 2bc7a510d..c2909437c 100644 --- a/composer.lock +++ b/composer.lock @@ -193,23 +193,24 @@ }, { "name": "mongodb/mongodb", - "version": "1.21.1", + "version": "2.1.0", "source": { "type": "git", "url": "https://github.com/mongodb/mongo-php-library.git", - "reference": "37bc8df3a67ddf8380704a5ba5dbd00e92ec1f6a" + "reference": "3bbe7ba9578724c7e1f47fcd17c881c0995baaad" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/37bc8df3a67ddf8380704a5ba5dbd00e92ec1f6a", - "reference": "37bc8df3a67ddf8380704a5ba5dbd00e92ec1f6a", + "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/3bbe7ba9578724c7e1f47fcd17c881c0995baaad", + "reference": "3bbe7ba9578724c7e1f47fcd17c881c0995baaad", "shasum": "" }, "require": { "composer-runtime-api": "^2.0", - "ext-mongodb": "^1.21.0", + "ext-mongodb": "^2.1", "php": "^8.1", - "psr/log": "^1.1.4|^2|^3" + "psr/log": "^1.1.4|^2|^3", + "symfony/polyfill-php85": "^1.32" }, "replace": { "mongodb/builder": "*" @@ -263,9 +264,9 @@ ], "support": { "issues": "https://github.com/mongodb/mongo-php-library/issues", - "source": "https://github.com/mongodb/mongo-php-library/tree/1.21.1" + "source": "https://github.com/mongodb/mongo-php-library/tree/2.1.0" }, - "time": "2025-02-28T17:24:20+00:00" + "time": "2025-05-23T10:48:05+00:00" }, { "name": "nyholm/psr7", @@ -1711,6 +1712,82 @@ ], "time": "2024-09-09T11:45:10+00:00" }, + { + "name": "symfony/polyfill-php85", + "version": "v1.32.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php85.git", + "reference": "6fedf31ce4e3648f4ff5ca58bfd53127d38f05fd" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/6fedf31ce4e3648f4ff5ca58bfd53127d38f05fd", + "reference": "6fedf31ce4e3648f4ff5ca58bfd53127d38f05fd", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php85\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.5+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php85/tree/v1.32.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2025-05-02T08:40:52+00:00" + }, { "name": "symfony/service-contracts", "version": "v3.6.0", @@ -1997,17 +2074,17 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "71c062c42bca6a4f709ddb86670089d5d4a5d7d5" + "reference": "e9ee5bce6262e1504c368c10a5ae7df2f3737a0e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/71c062c42bca6a4f709ddb86670089d5d4a5d7d5", - "reference": "71c062c42bca6a4f709ddb86670089d5d4a5d7d5", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/e9ee5bce6262e1504c368c10a5ae7df2f3737a0e", + "reference": "e9ee5bce6262e1504c368c10a5ae7df2f3737a0e", "shasum": "" }, "require": { "ext-mongodb": "*", - "mongodb/mongodb": "^1.21", + "mongodb/mongodb": "2.1.0", "php": ">=8.0" }, "require-dev": { @@ -2049,7 +2126,7 @@ "issues": "https://github.com/utopia-php/mongo/issues", "source": "https://github.com/utopia-php/mongo/tree/feat-mongo-transactions" }, - "time": "2025-07-14T08:20:00+00:00" + "time": "2025-07-21T10:12:18+00:00" }, { "name": "utopia-php/pools", @@ -2627,16 +2704,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.12.27", + "version": "1.12.28", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "3a6e423c076ab39dfedc307e2ac627ef579db162" + "reference": "fcf8b71aeab4e1a1131d1783cef97b23a51b87a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/3a6e423c076ab39dfedc307e2ac627ef579db162", - "reference": "3a6e423c076ab39dfedc307e2ac627ef579db162", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/fcf8b71aeab4e1a1131d1783cef97b23a51b87a9", + "reference": "fcf8b71aeab4e1a1131d1783cef97b23a51b87a9", "shasum": "" }, "require": { @@ -2681,7 +2758,7 @@ "type": "github" } ], - "time": "2025-05-21T20:51:45+00:00" + "time": "2025-07-17T17:15:39+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 62feb2323..d956b2666 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -535,6 +535,7 @@ public function createAttributes(string $collection, array $attributes): bool public function deleteAttribute(string $collection, string $id): bool { $collection = $this->getNamespace() . '_' . $this->filter($collection); + $this->getClient()->update( $collection, [], @@ -867,14 +868,14 @@ public function getDocument(string $collection, string $id, array $queries = [], } $result = $this->client->find($name, $filters, $options)->cursor->firstBatch; - + if (empty($result)) { return new Document([]); } $result = $this->replaceChars('_', '$', (array)$result[0]); $result = $this->timeToDocument($result); - + return new Document($result); } @@ -982,14 +983,14 @@ public function createDocuments(string $collection, array $documents): array */ private function insertDocument(string $name, array $document, array $options = []): array { - + try { $result = $this->client->insert($name, $document, $options); - - + + $filters = []; $filters['_uid'] = $document['_uid']; - + if ($this->sharedTables) { $filters['_tenant'] = $this->getTenant(); } @@ -999,12 +1000,12 @@ private function insertDocument(string $name, array $document, array $options = $name, $filters, array_merge($options, ['limit' => 1]) - )->cursor->firstBatch[0]; - - /** - * TODO Do we even need this find? - * We can just return the result from the insertDocument. - */ + )->cursor->firstBatch[0]; + + /** + * TODO Do we even need this find? + * We can just return the result from the insertDocument. + */ return $this->client->toArray($result); } catch (MongoException $e) { diff --git a/tests/e2e/Adapter/MongoDBTest.php b/tests/e2e/Adapter/MongoDBTest.php index 4f16b9581..55b21f8e4 100644 --- a/tests/e2e/Adapter/MongoDBTest.php +++ b/tests/e2e/Adapter/MongoDBTest.php @@ -9,9 +9,6 @@ use Utopia\Database\Adapter\Mongo; use Utopia\Database\Database; use Utopia\Mongo\Client; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; -use Utopia\Database\Document; class MongoDBTest extends Base { From 8cd9ac4fb3801410191f041ad3b6be75e5ce01b9 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 21 Jul 2025 13:48:02 +0300 Subject: [PATCH 06/16] updates --- composer.lock | 113 +++++++++++++++++++++++++++------ src/Database/Adapter/Mongo.php | 7 +- 2 files changed, 101 insertions(+), 19 deletions(-) diff --git a/composer.lock b/composer.lock index 2bc7a510d..c2909437c 100644 --- a/composer.lock +++ b/composer.lock @@ -193,23 +193,24 @@ }, { "name": "mongodb/mongodb", - "version": "1.21.1", + "version": "2.1.0", "source": { "type": "git", "url": "https://github.com/mongodb/mongo-php-library.git", - "reference": "37bc8df3a67ddf8380704a5ba5dbd00e92ec1f6a" + "reference": "3bbe7ba9578724c7e1f47fcd17c881c0995baaad" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/37bc8df3a67ddf8380704a5ba5dbd00e92ec1f6a", - "reference": "37bc8df3a67ddf8380704a5ba5dbd00e92ec1f6a", + "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/3bbe7ba9578724c7e1f47fcd17c881c0995baaad", + "reference": "3bbe7ba9578724c7e1f47fcd17c881c0995baaad", "shasum": "" }, "require": { "composer-runtime-api": "^2.0", - "ext-mongodb": "^1.21.0", + "ext-mongodb": "^2.1", "php": "^8.1", - "psr/log": "^1.1.4|^2|^3" + "psr/log": "^1.1.4|^2|^3", + "symfony/polyfill-php85": "^1.32" }, "replace": { "mongodb/builder": "*" @@ -263,9 +264,9 @@ ], "support": { "issues": "https://github.com/mongodb/mongo-php-library/issues", - "source": "https://github.com/mongodb/mongo-php-library/tree/1.21.1" + "source": "https://github.com/mongodb/mongo-php-library/tree/2.1.0" }, - "time": "2025-02-28T17:24:20+00:00" + "time": "2025-05-23T10:48:05+00:00" }, { "name": "nyholm/psr7", @@ -1711,6 +1712,82 @@ ], "time": "2024-09-09T11:45:10+00:00" }, + { + "name": "symfony/polyfill-php85", + "version": "v1.32.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php85.git", + "reference": "6fedf31ce4e3648f4ff5ca58bfd53127d38f05fd" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/6fedf31ce4e3648f4ff5ca58bfd53127d38f05fd", + "reference": "6fedf31ce4e3648f4ff5ca58bfd53127d38f05fd", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php85\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.5+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php85/tree/v1.32.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2025-05-02T08:40:52+00:00" + }, { "name": "symfony/service-contracts", "version": "v3.6.0", @@ -1997,17 +2074,17 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "71c062c42bca6a4f709ddb86670089d5d4a5d7d5" + "reference": "e9ee5bce6262e1504c368c10a5ae7df2f3737a0e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/71c062c42bca6a4f709ddb86670089d5d4a5d7d5", - "reference": "71c062c42bca6a4f709ddb86670089d5d4a5d7d5", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/e9ee5bce6262e1504c368c10a5ae7df2f3737a0e", + "reference": "e9ee5bce6262e1504c368c10a5ae7df2f3737a0e", "shasum": "" }, "require": { "ext-mongodb": "*", - "mongodb/mongodb": "^1.21", + "mongodb/mongodb": "2.1.0", "php": ">=8.0" }, "require-dev": { @@ -2049,7 +2126,7 @@ "issues": "https://github.com/utopia-php/mongo/issues", "source": "https://github.com/utopia-php/mongo/tree/feat-mongo-transactions" }, - "time": "2025-07-14T08:20:00+00:00" + "time": "2025-07-21T10:12:18+00:00" }, { "name": "utopia-php/pools", @@ -2627,16 +2704,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.12.27", + "version": "1.12.28", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "3a6e423c076ab39dfedc307e2ac627ef579db162" + "reference": "fcf8b71aeab4e1a1131d1783cef97b23a51b87a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/3a6e423c076ab39dfedc307e2ac627ef579db162", - "reference": "3a6e423c076ab39dfedc307e2ac627ef579db162", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/fcf8b71aeab4e1a1131d1783cef97b23a51b87a9", + "reference": "fcf8b71aeab4e1a1131d1783cef97b23a51b87a9", "shasum": "" }, "require": { @@ -2681,7 +2758,7 @@ "type": "github" } ], - "time": "2025-05-21T20:51:45+00:00" + "time": "2025-07-17T17:15:39+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 62feb2323..b343ed073 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -89,6 +89,11 @@ public function clearTimeout(string $event): void */ public function withTransaction(callable $callback): mixed { + // If the database is not a replica set, we can't use transactions + if(!$this->client->isReplicaSet()){ + return true; + } + // Removed the attmpts to retry the transaction. //Unlike pdo if we run theabortTransaction more then once (same transactioId), // it will throw an error the there is no transaction in progress. @@ -1162,7 +1167,7 @@ public function createOrUpdateDocuments(string $collection, string $attribute, a } $options = $this->addTransactionContext([]); - $this->client->bulkUpsert( + $this->client->upsert( $name, $operations, options: $options From 142d167e8ab62bc6fd8568aee9115de364f5b5dd Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 21 Jul 2025 16:58:51 +0300 Subject: [PATCH 07/16] updates --- src/Database/Adapter/Mongo.php | 42 ++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index b97ada4fa..5069dbf27 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -153,20 +153,21 @@ public function commitTransaction(): bool { try { if ($this->inTransaction === 0) { - throw new DatabaseException('No transaction in progress'); + return false; } $this->inTransaction--; if ($this->inTransaction === 0) { if (!$this->sessionId) { - throw new DatabaseException('No session in progress'); + return false; } - $result = $this->client->commitTransaction( - ['id' => $this->sessionId], // Pass raw id object - $this->txnNumber, - false - ); - if (($result->ok ?? 0) !== 1.0) { - throw new DatabaseException('Failed to commit transaction'); + try { + $result = $this->client->commitTransaction( + ['id' => $this->sessionId], // Pass raw id object + $this->txnNumber, + false + ); + } catch (\Throwable $e) { + throw new DatabaseException($e->getMessage(), $e->getCode(), $e); } // Session is now closed by the client using endSessions, state is reseted @@ -195,13 +196,14 @@ public function rollbackTransaction(): bool throw new DatabaseException('No session in progress'); } - $result = $this->client->abortTransaction( - ['id' => $this->sessionId], // Pass raw id object - $this->txnNumber, - false - ); - if (($result->ok ?? 0) !== 1.0) { - throw new DatabaseException('Failed to rollback transaction'); + try { + $result = $this->client->abortTransaction( + ['id' => $this->sessionId], // Pass raw id object + $this->txnNumber, + false + ); + } catch (\Throwable $e) { + throw new DatabaseException($e->getMessage(), $e->getCode(), $e); } // Session is now closed by the client using endSessions, reset our state @@ -936,7 +938,6 @@ public function createDocuments(string $collection, array $documents): array $name = $this->getNamespace() . '_' . $this->filter($collection); $options = $this->addTransactionContext([]); - $records = []; $hasSequence = null; $documents = array_map(fn ($doc) => clone $doc, $documents); @@ -991,8 +992,6 @@ private function insertDocument(string $name, array $document, array $options = try { $result = $this->client->insert($name, $document, $options); - - $filters = []; $filters['_uid'] = $document['_uid']; @@ -1070,6 +1069,7 @@ public function updateDocuments(string $collection, Document $updates, array $do { $name = $this->getNamespace() . '_' . $this->filter($collection); + $options = $this->addTransactionContext([]); $queries = [ Query::equal('$sequence', \array_map(fn ($document) => $document->getSequence(), $documents)) ]; @@ -1089,7 +1089,6 @@ public function updateDocuments(string $collection, Document $updates, array $do ]; try { - $options = $this->addTransactionContext([]); $this->client->update($name, $filters, $updateQuery, multi: true, options: $options); } catch (MongoException $e) { throw new Duplicate($e->getMessage()); @@ -1112,6 +1111,7 @@ public function createOrUpdateDocuments(string $collection, string $attribute, a try { $name = $this->getNamespace() . '_' . $this->filter($collection); + $attribute = $this->filter($attribute); $documentIds = []; @@ -1168,6 +1168,7 @@ public function createOrUpdateDocuments(string $collection, string $attribute, a } $options = $this->addTransactionContext([]); + $this->client->upsert( $name, $operations, @@ -1401,6 +1402,7 @@ protected function getInternalKeyForAttribute(string $attribute): string public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, string $forPermission = Database::PERMISSION_READ): array { $name = $this->getNamespace() . '_' . $this->filter($collection); + $queries = array_map(fn ($query) => clone $query, $queries); $filters = $this->buildFilters($queries); From e6b2bba2ae38b29bcd8e699039ac3fc407ba0c2a Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 21 Jul 2025 17:02:17 +0300 Subject: [PATCH 08/16] updates --- src/Database/Adapter/Mongo.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 5069dbf27..2169ab0f8 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -188,12 +188,12 @@ public function rollbackTransaction(): bool try { if ($this->inTransaction === 0) { - throw new DatabaseException('No transaction in progress'); + return false; } $this->inTransaction--; if ($this->inTransaction === 0) { if (!$this->sessionId) { - throw new DatabaseException('No session in progress'); + return false; } try { @@ -1402,7 +1402,7 @@ protected function getInternalKeyForAttribute(string $attribute): string public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, string $forPermission = Database::PERMISSION_READ): array { $name = $this->getNamespace() . '_' . $this->filter($collection); - + $queries = array_map(fn ($query) => clone $query, $queries); $filters = $this->buildFilters($queries); From ec67b5946bedde271bf3697b5587013c0740c259 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 21 Jul 2025 17:06:13 +0300 Subject: [PATCH 09/16] updates --- src/Database/Adapter/Mongo.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 2169ab0f8..12e1ca778 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -348,6 +348,7 @@ public function createCollection(string $name, array $attributes = [], array $in // Returns an array/object with the result document try { $this->getClient()->createCollection($id); + } catch (MongoException $e) { throw new Duplicate($e->getMessage(), $e->getCode(), $e); } @@ -1111,7 +1112,6 @@ public function createOrUpdateDocuments(string $collection, string $attribute, a try { $name = $this->getNamespace() . '_' . $this->filter($collection); - $attribute = $this->filter($attribute); $documentIds = []; @@ -1322,7 +1322,8 @@ public function deleteDocuments(string $collection, array $sequences, array $per $filters = $this->replaceInternalIdsKeys($filters, '$', '_', $this->operators); $filters = $this->timeFilter($filters); - $options = $this->addTransactionContext([]); + + $options = []; try { $count = $this->client->delete( @@ -1402,7 +1403,6 @@ protected function getInternalKeyForAttribute(string $attribute): string public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, string $forPermission = Database::PERMISSION_READ): array { $name = $this->getNamespace() . '_' . $this->filter($collection); - $queries = array_map(fn ($query) => clone $query, $queries); $filters = $this->buildFilters($queries); From 1760ea5674d9986455d1a0eb291c3346ee34ac5d Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 24 Jul 2025 18:57:17 +0300 Subject: [PATCH 10/16] sunc against upsert pr --- src/Database/Adapter/Mongo.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index fa191d10c..08f501fb8 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -90,7 +90,7 @@ public function clearTimeout(string $event): void public function withTransaction(callable $callback): mixed { // If the database is not a replica set, we can't use transactions - if(!$this->client->isReplicaSet()){ + if (!$this->client->isReplicaSet()) { return true; } From 9edd05fd127d47a9da564b0875e67e93b4bc0792 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 24 Jul 2025 23:48:54 +0300 Subject: [PATCH 11/16] Update composer.lock and docker-compose.yml for dependency versions and MongoDB configuration adjustments --- composer.lock | 12 ++++++------ docker-compose.yml | 16 +++++++++++----- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/composer.lock b/composer.lock index c2909437c..7d0e83076 100644 --- a/composer.lock +++ b/composer.lock @@ -2074,23 +2074,23 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "e9ee5bce6262e1504c368c10a5ae7df2f3737a0e" + "reference": "68dcddf5f266822f4a5b361a1aeafa5a9036d4bf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/e9ee5bce6262e1504c368c10a5ae7df2f3737a0e", - "reference": "e9ee5bce6262e1504c368c10a5ae7df2f3737a0e", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/68dcddf5f266822f4a5b361a1aeafa5a9036d4bf", + "reference": "68dcddf5f266822f4a5b361a1aeafa5a9036d4bf", "shasum": "" }, "require": { - "ext-mongodb": "*", + "ext-mongodb": "2.1.1", "mongodb/mongodb": "2.1.0", "php": ">=8.0" }, "require-dev": { "fakerphp/faker": "^1.14", "laravel/pint": "1.2.*", - "phpstan/phpstan": "1.8.*", + "phpstan/phpstan": "2.1.*", "phpunit/phpunit": "^9.4", "swoole/ide-helper": "4.8.0" }, @@ -2126,7 +2126,7 @@ "issues": "https://github.com/utopia-php/mongo/issues", "source": "https://github.com/utopia-php/mongo/tree/feat-mongo-transactions" }, - "time": "2025-07-21T10:12:18+00:00" + "time": "2025-07-24T20:15:02+00:00" }, { "name": "utopia-php/pools", diff --git a/docker-compose.yml b/docker-compose.yml index 26911a3af..993f78065 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -85,11 +85,17 @@ services: environment: MONGO_INITDB_DATABASE: utopia_testing MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: password - command: > - mongod --replSet rs0 - --auth - --keyFile /etc/mongo-keyfile + MONGO_INITDB_ROOT_PASSWORD: password + # Replica set + # MONGO_INITDB_REPLICA_SET: rs0 + # MONGO_INITDB_REPLICA_SET_NAME: rs0 + # MONGO_INITDB_REPLICA_SET_KEY: rs0 + # MONGO_INITDB_REPLICA_SET_KEY_FILE: /etc/mongo-keyfile + # MONGO_INITDB_REPLICA_SET_KEY_FILE_NAME: mongo-keyfile + # MONGO_INITDB_REPLICA_SET_KEY_FILE_PATH: /etc/mongo-keyfile + # MONGO_INITDB_REPLICA_SET_KEY_FILE_PATH_NAME: mongo-keyfile + # MONGO_INITDB_REPLICA_SET_KEY_FILE_PATH_NAME_NAME: mongo-keyfile + # Manyally initate the replica set #docker compose exec mongo mongosh admin -u root -p password --eval 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongo:27017"}]})' From 85748b7fbc79a4382d0e871ade20c27d1342df17 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 27 Jul 2025 16:16:53 +0300 Subject: [PATCH 12/16] sync with feat-mongo-2 --- composer.lock | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/composer.lock b/composer.lock index 7d0e83076..8598421d1 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "a3b0ff08e6addea30a6380a0b19a0529", + "content-hash": "f6dc7d44d9bb06432e3a2d2bf026022a", "packages": [ { "name": "brick/math", @@ -2070,16 +2070,16 @@ }, { "name": "utopia-php/mongo", - "version": "dev-feat-mongo-transactions", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "68dcddf5f266822f4a5b361a1aeafa5a9036d4bf" + "reference": "b7a4901f552f6383b274d5a6c84feba6357afa95" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/68dcddf5f266822f4a5b361a1aeafa5a9036d4bf", - "reference": "68dcddf5f266822f4a5b361a1aeafa5a9036d4bf", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/b7a4901f552f6383b274d5a6c84feba6357afa95", + "reference": "b7a4901f552f6383b274d5a6c84feba6357afa95", "shasum": "" }, "require": { @@ -2124,9 +2124,9 @@ ], "support": { "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/feat-mongo-transactions" + "source": "https://github.com/utopia-php/mongo/tree/0.5.0" }, - "time": "2025-07-24T20:15:02+00:00" + "time": "2025-07-25T04:02:37+00:00" }, { "name": "utopia-php/pools", @@ -4335,18 +4335,9 @@ "time": "2022-10-09T10:19:07+00:00" } ], - "aliases": [ - { - "package": "utopia-php/mongo", - "version": "dev-feat-mongo-transactions", - "alias": "0.3.1", - "alias_normalized": "0.3.1.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/mongo": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 4b24c1391bd7895fbb6ad80c7cfb87e04439ea8c Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 27 Jul 2025 16:18:23 +0300 Subject: [PATCH 13/16] sync with feat-mongo-2 --- src/Database/Adapter/Mongo.php | 4 ++-- src/Database/Database.php | 4 ++-- tests/e2e/Adapter/Scopes/IndexTests.php | 2 +- tests/e2e/Adapter/Scopes/PermissionTests.php | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 93ff3ba3b..ea66f8e59 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1322,7 +1322,7 @@ public function createOrUpdateDocuments(string $collection, string $attribute, a $tempDocuments[] = $change->getNew(); } } - + $sequences = $this->getSequences($collection, $tempDocuments); foreach ($changes as $change) { @@ -1354,7 +1354,7 @@ public function getSequences(string $collection, array $documents): array foreach ($documents as $document) { if (empty($document->getSequence())) { $documentIds[] = $document->getId(); - + if ($this->sharedTables) { $documentTenants[] = $document->getTenant(); } diff --git a/src/Database/Database.php b/src/Database/Database.php index 631fb83cd..dce84737a 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5079,14 +5079,14 @@ public function createOrUpdateDocumentsWithIncrease( /** * @var array $chunk */ - + $batch = $this->withTransaction(fn () => Authorization::skip(fn () => $this->adapter->createOrUpdateDocuments( $collection->getId(), $attribute, $chunk ))); $batch = $this->adapter->getSequences($collection->getId(), $batch); - + foreach ($chunk as $change) { if ($change->getOld()->isEmpty()) { $created++; diff --git a/tests/e2e/Adapter/Scopes/IndexTests.php b/tests/e2e/Adapter/Scopes/IndexTests.php index 1d40c553e..df3207f35 100644 --- a/tests/e2e/Adapter/Scopes/IndexTests.php +++ b/tests/e2e/Adapter/Scopes/IndexTests.php @@ -304,7 +304,7 @@ public function testIndexLengthZero(): void $database = static::getDatabase(); $database->createCollection(__FUNCTION__); - + $database->createAttribute(__FUNCTION__, 'title1', Database::VAR_STRING, $database->getAdapter()->getMaxIndexLength() + 300, true); try { diff --git a/tests/e2e/Adapter/Scopes/PermissionTests.php b/tests/e2e/Adapter/Scopes/PermissionTests.php index 97ebc8de1..285ff2e4c 100644 --- a/tests/e2e/Adapter/Scopes/PermissionTests.php +++ b/tests/e2e/Adapter/Scopes/PermissionTests.php @@ -759,7 +759,7 @@ public function testCollectionPermissionsRelationshipsFindWorks(array $data): vo $this->expectNotToPerformAssertions(); return; } - + $documents = $database->find( $collection->getId() ); From 5bcf41ed990b36400663b6f301b255f176d98299 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 27 Jul 2025 16:58:07 +0300 Subject: [PATCH 14/16] Refactor MongoDB configuration in docker-compose and enhance transaction handling in Mongo adapter. Updated command for MongoDB to support replica sets and improved transaction callback handling in the adapter. --- docker-compose.yml | 12 ++---------- src/Database/Adapter/Mongo.php | 3 ++- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2f9b8301c..8b64c9822 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,16 +87,8 @@ services: environment: MONGO_INITDB_DATABASE: utopia_testing MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: password - # Replica set - # MONGO_INITDB_REPLICA_SET: rs0 - # MONGO_INITDB_REPLICA_SET_NAME: rs0 - # MONGO_INITDB_REPLICA_SET_KEY: rs0 - # MONGO_INITDB_REPLICA_SET_KEY_FILE: /etc/mongo-keyfile - # MONGO_INITDB_REPLICA_SET_KEY_FILE_NAME: mongo-keyfile - # MONGO_INITDB_REPLICA_SET_KEY_FILE_PATH: /etc/mongo-keyfile - # MONGO_INITDB_REPLICA_SET_KEY_FILE_PATH_NAME: mongo-keyfile - # MONGO_INITDB_REPLICA_SET_KEY_FILE_PATH_NAME_NAME: mongo-keyfile + MONGO_INITDB_ROOT_PASSWORD: password + command: mongod --replSet rs0 --bind_ip_all --keyFile /etc/mongo-keyfile # Manyally initate the replica set #docker compose exec mongo mongosh admin -u root -p password --eval 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongo:27017"}]})' diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index ea66f8e59..293322fa9 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -91,7 +91,8 @@ public function withTransaction(callable $callback): mixed { // If the database is not a replica set, we can't use transactions if (!$this->client->isReplicaSet()) { - return true; + $result = $callback(); + return $result; } // Removed the attmpts to retry the transaction. From f498894a1242aa8e0e4cb3b78f3a3e1ef003e692 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 29 Jul 2025 16:16:28 +0300 Subject: [PATCH 15/16] Update docker-compose.yml to add a note about manual initiation of the MongoDB replica set and user creation. --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 9c3d98bf7..f2da3f9b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,6 +91,7 @@ services: command: mongod --replSet rs0 --bind_ip_all --keyFile /etc/mongo-keyfile # Manyally initate the replica set +# mongo users(!root) do not get created automatically!!! #docker compose exec mongo mongosh admin -u root -p password --eval 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongo:27017"}]})' mongo-express: From 2b65727a851a11671983faa22f6ebc582d99f26c Mon Sep 17 00:00:00 2001 From: Shimon Newman Date: Sun, 3 Aug 2025 10:11:52 +0300 Subject: [PATCH 16/16] Update src/Database/Adapter/Mongo.php Co-authored-by: Jake Barnby --- src/Database/Adapter/Mongo.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index d2af6b69f..7d80d7af0 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -225,7 +225,6 @@ public function rollbackTransaction(): bool */ private function addTransactionContext(array $options = []): array { - if ($this->inTransaction) { $options['lsid'] = ['id' => $this->sessionId]; $options['txnNumber'] = new \MongoDB\BSON\Int64($this->txnNumber);