From 5c4e6c61c393d176348e88b65730a14b52f4ea2e Mon Sep 17 00:00:00 2001 From: Darshan Date: Tue, 8 Apr 2025 14:08:41 +0530 Subject: [PATCH 01/14] feat: csv source [WIP]. --- src/Migration/Sources/Csv.php | 153 ++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 src/Migration/Sources/Csv.php diff --git a/src/Migration/Sources/Csv.php b/src/Migration/Sources/Csv.php new file mode 100644 index 00000000..b7952d58 --- /dev/null +++ b/src/Migration/Sources/Csv.php @@ -0,0 +1,153 @@ +$filePath = $filePath; + $this->resourceId = $resourceId; + $this->deviceForFiles = $deviceForFiles; + } + + public static function getName(): string + { + return 'Csv'; + } + + public static function getSupportedResources(): array + { + return [ + Resource::TYPE_DOCUMENT, + ]; + } + + public function report(array $resources = []): array + { + return []; + } + + protected function exportGroupAuth(int $batchSize, array $resources): void + { + throw new \Exception('Not Implemented'); + } + + protected function exportGroupDatabases(int $batchSize, array $resources): void + { + try { + if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { + $this->exportDocuments($batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + Resource::TYPE_DOCUMENT, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + } + } + + private function exportDocuments(int $batchSize): void + { + if (! $this->deviceForFiles->exists($this->filePath)) { + return; + } + + [$databaseId, $collectionId] = explode(':', $this->resourceId); + + $csvFileSource = $this->deviceForFiles->read($this->filePath); + + $file = fopen($csvFileSource, 'r'); + if (! $file) { + return; + } + + $headers = fgetcsv($file); + if (! is_array($headers)) { + fclose($file); + return; + } + + $buffer = []; + + while (($row = fgetcsv($file)) !== false) { + $data = array_combine($headers, $row); + if ($data === false) { + continue; + } + + $docId = $data['$id'] ?? 'unique()'; + $database = new Database($databaseId, ''); + $collection = new Collection($database, '', $collectionId); + $document = new Document($docId, $collection, $data); + + echo "CSV Row:\n"; + var_dump($data); + + echo "Document:\n"; + var_dump($document); + + $buffer[] = $document; + + if (count($buffer) === $batchSize) { + $this->callback($buffer); + $buffer = []; + } + } + + fclose($file); + + if (! empty($buffer)) { + $this->callback($buffer); + } + } + + protected function exportGroupStorage(int $batchSize, array $resources): void + { + throw new \Exception('Not Implemented'); + } + + protected function exportBuckets(int $batchSize): void + { + throw new \Exception('Not Implemented'); + } + + private function exportFiles(int $batchSize): void + { + throw new \Exception('Not Implemented'); + } + + private function exportFile(File $file): void + { + throw new \Exception('Not Implemented'); + } + + protected function exportGroupFunctions(int $batchSize, array $resources): void + { + throw new \Exception('Not Implemented'); + } +} From c8a70fc2da98821d9d855e4d543bff588f703fda Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 9 Apr 2025 11:42:07 +0530 Subject: [PATCH 02/14] update: improve and fix things. --- src/Migration/Sources/Csv.php | 47 +++++++++++++++++------------------ 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/src/Migration/Sources/Csv.php b/src/Migration/Sources/Csv.php index b7952d58..757b3a1c 100644 --- a/src/Migration/Sources/Csv.php +++ b/src/Migration/Sources/Csv.php @@ -2,6 +2,7 @@ namespace Utopia\Migration\Sources; +use Utopia\CLI\Console; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Database\Collection; @@ -21,13 +22,13 @@ class Csv extends Source */ public string $resourceId; - public Device $deviceForFiles; + public Device $deviceForLocal; - public function __construct(string $resourceId, string $filePath, Device $deviceForFiles) + public function __construct(string $resourceId, string $filePath, Device $deviceForLocal) { - $this->$filePath = $filePath; + $this->filePath = $filePath; $this->resourceId = $resourceId; - $this->deviceForFiles = $deviceForFiles; + $this->deviceForLocal = $deviceForLocal; } public static function getName(): string @@ -68,49 +69,47 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void previous: $e ) ); + } finally { + // delete the temporary file! + // temporary logs. + Console::log('File exists: '.$this->deviceForLocal->exists($this->filePath)); + $this->deviceForLocal->delete($this->filePath); + Console::log('File exists: '.$this->deviceForLocal->exists($this->filePath)); } } private function exportDocuments(int $batchSize): void { - if (! $this->deviceForFiles->exists($this->filePath)) { + if (! $this->deviceForLocal->exists($this->filePath)) { return; } - [$databaseId, $collectionId] = explode(':', $this->resourceId); - - $csvFileSource = $this->deviceForFiles->read($this->filePath); - - $file = fopen($csvFileSource, 'r'); - if (! $file) { + $stream = fopen($this->filePath, 'r'); + if (! $stream) { return; } - $headers = fgetcsv($file); - if (! is_array($headers)) { - fclose($file); + $headers = fgetcsv($stream); + if (! is_array($headers) || count($headers) === 0) { + fclose($stream); return; } + [$databaseId, $collectionId] = explode(':', $this->resourceId); + // TODO: @itznotabug, @jake - do we need to check for permissions here or db handles it? + $collection = new Collection(new Database($databaseId, ''), '', $collectionId); + $buffer = []; - while (($row = fgetcsv($file)) !== false) { + while (($row = fgetcsv($stream)) !== false) { $data = array_combine($headers, $row); if ($data === false) { continue; } $docId = $data['$id'] ?? 'unique()'; - $database = new Database($databaseId, ''); - $collection = new Collection($database, '', $collectionId); $document = new Document($docId, $collection, $data); - echo "CSV Row:\n"; - var_dump($data); - - echo "Document:\n"; - var_dump($document); - $buffer[] = $document; if (count($buffer) === $batchSize) { @@ -119,7 +118,7 @@ private function exportDocuments(int $batchSize): void } } - fclose($file); + fclose($stream); if (! empty($buffer)) { $this->callback($buffer); From 387541eb25e7fd68ef59b3026e2604ec8ec34220 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 9 Apr 2025 16:31:19 +0530 Subject: [PATCH 03/14] update: improve and fix things. --- src/Migration/Sources/Csv.php | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/Migration/Sources/Csv.php b/src/Migration/Sources/Csv.php index 757b3a1c..b8dab588 100644 --- a/src/Migration/Sources/Csv.php +++ b/src/Migration/Sources/Csv.php @@ -2,7 +2,6 @@ namespace Utopia\Migration\Sources; -use Utopia\CLI\Console; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Database\Collection; @@ -22,13 +21,13 @@ class Csv extends Source */ public string $resourceId; - public Device $deviceForLocal; + public Device $deviceForCsvImports; - public function __construct(string $resourceId, string $filePath, Device $deviceForLocal) + public function __construct(string $resourceId, string $filePath, Device $deviceForCsvImports) { $this->filePath = $filePath; $this->resourceId = $resourceId; - $this->deviceForLocal = $deviceForLocal; + $this->deviceForCsvImports = $deviceForCsvImports; } public static function getName(): string @@ -71,16 +70,13 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void ); } finally { // delete the temporary file! - // temporary logs. - Console::log('File exists: '.$this->deviceForLocal->exists($this->filePath)); - $this->deviceForLocal->delete($this->filePath); - Console::log('File exists: '.$this->deviceForLocal->exists($this->filePath)); + $this->deviceForCsvImports->delete($this->filePath); } } private function exportDocuments(int $batchSize): void { - if (! $this->deviceForLocal->exists($this->filePath)) { + if (! $this->deviceForCsvImports->exists($this->filePath)) { return; } From b308d9183f1f8ab32e172f31744ad93db319cba9 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 9 Apr 2025 18:24:03 +0530 Subject: [PATCH 04/14] update: pagination and attribute type handling. --- src/Migration/Sources/Csv.php | 119 ++++++++++++++++++++++++++++++++-- 1 file changed, 112 insertions(+), 7 deletions(-) diff --git a/src/Migration/Sources/Csv.php b/src/Migration/Sources/Csv.php index b8dab588..ec0d5287 100644 --- a/src/Migration/Sources/Csv.php +++ b/src/Migration/Sources/Csv.php @@ -2,32 +2,43 @@ namespace Utopia\Migration\Sources; +use Utopia\Database\Database as UtopiaDatabase; use Utopia\Migration\Exception; use Utopia\Migration\Resource; +use Utopia\Migration\Resources\Database\Attribute; use Utopia\Migration\Resources\Database\Collection; use Utopia\Migration\Resources\Database\Database; use Utopia\Migration\Resources\Database\Document; use Utopia\Migration\Resources\Storage\File; use Utopia\Migration\Source; +use Utopia\Migration\Sources\Appwrite\Reader; +use Utopia\Migration\Sources\Appwrite\Reader\Database as DatabaseReader; use Utopia\Migration\Transfer; use Utopia\Storage\Device; class Csv extends Source { - public string $filePath; + private string $filePath; /** * format: `{databaseId:collectionId}` */ - public string $resourceId; + private string $resourceId; - public Device $deviceForCsvImports; + private Device $deviceForCsvImports; - public function __construct(string $resourceId, string $filePath, Device $deviceForCsvImports) - { + private Reader $database; + + public function __construct( + string $resourceId, + string $filePath, + Device $deviceForCsvImports, + ?UtopiaDatabase $dbForProject + ) { $this->filePath = $filePath; $this->resourceId = $resourceId; $this->deviceForCsvImports = $deviceForCsvImports; + $this->database = new DatabaseReader($dbForProject); } public static function getName(): string @@ -74,6 +85,9 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void } } + /** + * @throws Exception|\Utopia\Database\Exception + */ private function exportDocuments(int $batchSize): void { if (! $this->deviceForCsvImports->exists($this->filePath)) { @@ -91,10 +105,58 @@ private function exportDocuments(int $batchSize): void return; } + $allAttributes = []; + $lastAttribute = null; + [$databaseId, $collectionId] = explode(':', $this->resourceId); // TODO: @itznotabug, @jake - do we need to check for permissions here or db handles it? $collection = new Collection(new Database($databaseId, ''), '', $collectionId); + while (true) { + // paginate over the attributes + $queries = [$this->database->queryLimit($batchSize)]; + if ($lastAttribute) { + $queries[] = $this->database->queryCursorAfter($lastAttribute); + } + + $fetched = $this->database->listAttributes($collection, $queries); + if (empty($fetched)) { + break; + } + + $allAttributes = array_merge($allAttributes, $fetched); + $lastAttribute = $fetched[count($fetched) - 1]; + + if (count($fetched) < $batchSize) { + break; + } + } + + $attributeTypes = []; + $manyToManyKeys = []; + + foreach ($allAttributes as $attribute) { + $key = $attribute['key']; + + if ( + // Skip child-side relationships entirely + $attribute['type'] === Attribute::TYPE_RELATIONSHIP && + ($attribute['side'] ?? '') === UtopiaDatabase::RELATION_SIDE_CHILD + ) { + continue; + } + + $attributeTypes[$key] = $attribute['type']; + + if ( + $attribute['type'] === Attribute::TYPE_RELATIONSHIP && + ($attribute['relationType'] ?? '') === 'manyToMany' && + ($attribute['side'] ?? '') === 'parent' + ) { + $manyToManyKeys[] = $key; + } + } + $buffer = []; while (($row = fgetcsv($stream)) !== false) { @@ -103,8 +165,51 @@ private function exportDocuments(int $batchSize): void continue; } - $docId = $data['$id'] ?? 'unique()'; - $document = new Document($docId, $collection, $data); + $parsedData = $data; + + foreach ($data as $key => $value) { + if (! isset($attributeTypes[$key])) { + continue; + } + + $type = $attributeTypes[$key]; + $parsedValue = trim($value); + + if ($parsedValue === '') { + $parsedData[$key] = null; + + continue; + } + + // TODO: @itznotabug, @jake - should we support Relationships like these? + if (in_array($key, $manyToManyKeys, true)) { + $parsedData[$key] = str_contains($parsedValue, ',') + ? array_map('trim', explode(',', $parsedValue)) + : [$parsedValue]; + + continue; + } + + switch ($type) { + case Attribute::TYPE_INTEGER: + $parsedData[$key] = is_numeric($parsedValue) ? (int) $parsedValue : null; + break; + + case Attribute::TYPE_FLOAT: + $parsedData[$key] = is_numeric($parsedValue) ? (float) $parsedValue : null; + break; + + case Attribute::TYPE_BOOLEAN: + $parsedData[$key] = filter_var($parsedValue, FILTER_VALIDATE_BOOLEAN); + break; + + default: + break; + } + } + + $docId = $parsedData['$id'] ?? 'unique()'; + $document = new Document($docId, $collection, $parsedData); $buffer[] = $document; From c1b0ddd1f7150a8e8be251de49d05c27780c8ffd Mon Sep 17 00:00:00 2001 From: Darshan Date: Sat, 12 Apr 2025 16:42:01 +0530 Subject: [PATCH 05/14] update: support for `$permissions`. --- src/Migration/Sources/Csv.php | 99 ++++++++++++++++++++++++++++++++--- 1 file changed, 92 insertions(+), 7 deletions(-) diff --git a/src/Migration/Sources/Csv.php b/src/Migration/Sources/Csv.php index ec0d5287..789af107 100644 --- a/src/Migration/Sources/Csv.php +++ b/src/Migration/Sources/Csv.php @@ -3,6 +3,7 @@ namespace Utopia\Migration\Sources; use Utopia\Database\Database as UtopiaDatabase; +use Utopia\Database\Document as UtopiaDocument; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Database\Attribute; @@ -29,6 +30,8 @@ class Csv extends Source private Reader $database; + private ?UtopiaDatabase $dbForProject; + public function __construct( string $resourceId, string $filePath, @@ -38,6 +41,8 @@ public function __construct( $this->filePath = $filePath; $this->resourceId = $resourceId; $this->deviceForCsvImports = $deviceForCsvImports; + + $this->dbForProject = $dbForProject; $this->database = new DatabaseReader($dbForProject); } @@ -58,6 +63,9 @@ public function report(array $resources = []): array return []; } + /** + * @throws \Exception + */ protected function exportGroupAuth(int $batchSize, array $resources): void { throw new \Exception('Not Implemented'); @@ -86,7 +94,7 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void } /** - * @throws Exception|\Utopia\Database\Exception + * @throws \Exception */ private function exportDocuments(int $batchSize): void { @@ -109,11 +117,13 @@ private function exportDocuments(int $batchSize): void $lastAttribute = null; [$databaseId, $collectionId] = explode(':', $this->resourceId); - // TODO: @itznotabug, @jake - do we need to check for permissions here or db handles it? - $collection = new Collection(new Database($databaseId, ''), '', $collectionId); + $database = new Database($databaseId, ''); + $collection = new Collection($database, '', $collectionId); + + $collectionStructure = $this->getCollection($databaseId, $collectionId); + $hasDocumentSecurityEnabled = $collectionStructure->getAttribute('documentSecurity', false); while (true) { - // paginate over the attributes $queries = [$this->database->queryLimit($batchSize)]; if ($lastAttribute) { $queries[] = $this->database->queryCursorAfter($lastAttribute); @@ -181,7 +191,6 @@ private function exportDocuments(int $batchSize): void continue; } - // TODO: @itznotabug, @jake - should we support Relationships like these? if (in_array($key, $manyToManyKeys, true)) { $parsedData[$key] = str_contains($parsedValue, ',') ? array_map('trim', explode(',', $parsedValue)) @@ -208,9 +217,20 @@ private function exportDocuments(int $batchSize): void } } - $docId = $parsedData['$id'] ?? 'unique()'; - $document = new Document($docId, $collection, $parsedData); + $permissions = []; + $documentId = $parsedData['$id'] ?? 'unique()'; + + if ($hasDocumentSecurityEnabled && isset($parsedData['$permissions'])) { + $permissions = $this->parsePermissions($parsedData['$permissions']); + } + + foreach ($parsedData as $key => $value) { + if (str_starts_with($key, '$')) { + unset($parsedData[$key]); + } + } + $document = new Document($documentId, $collection, $parsedData, $permissions); $buffer[] = $document; if (count($buffer) === $batchSize) { @@ -226,26 +246,91 @@ private function exportDocuments(int $batchSize): void } } + /** + * Fast path function without the built-in `listCollections` for better performance! + * + * @throws \Exception + */ + public function getCollection(string $databaseId, string $collectionId): UtopiaDocument + { + $database = $this->dbForProject->getDocument('databases', $databaseId); + if ($database->isEmpty()) { + return new UtopiaDocument; + } + + return $this->dbForProject->getDocument('database_'.$database->getInternalId(), $collectionId); + } + + /** + * Parses a stringified permission array into a string[]. + * + * Example: + * ``` + * "[read(\"user:user1234\"),read(\"user:user4321\")]" + * ``` + * Into: + * ``` + * [ + * "read(\"user:user1234\")", + * "read(\"user:user4321\")" + * ] + * ``` + * + * @param string $raw + * @return string[] + */ + private function parsePermissions(string $raw): array + { + $raw = trim($raw, ' "[]'); + + if (empty($raw)) { + return []; + } + + $parts = preg_split('/,(?![^(]*\))/', $raw); + + return array_map(function ($item) { + $item = trim($item); + + return str_replace('\"', '"', $item); + }, $parts); + } + + /** + * @throws \Exception + */ protected function exportGroupStorage(int $batchSize, array $resources): void { throw new \Exception('Not Implemented'); } + /** + * @throws \Exception + */ protected function exportBuckets(int $batchSize): void { throw new \Exception('Not Implemented'); } + /** + * @throws \Exception + */ private function exportFiles(int $batchSize): void { throw new \Exception('Not Implemented'); } + /** + * @throws \Exception + */ private function exportFile(File $file): void { throw new \Exception('Not Implemented'); } + /** + * @throws \Exception + */ protected function exportGroupFunctions(int $batchSize, array $resources): void { throw new \Exception('Not Implemented'); From 9847a1387136574c7539872232765c5c4d8064f7 Mon Sep 17 00:00:00 2001 From: Darshan Date: Sat, 12 Apr 2025 16:42:09 +0530 Subject: [PATCH 06/14] bump: composer deps. --- composer.lock | 60 +++++++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/composer.lock b/composer.lock index c7356580..6e04f416 100644 --- a/composer.lock +++ b/composer.lock @@ -635,16 +635,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.2.2", + "version": "1.2.3", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "37eec0fe47ddd627911f318f29b6cd48196be0c0" + "reference": "0e7804c176c4b09d95b7985400aa38ce544cb7fc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/37eec0fe47ddd627911f318f29b6cd48196be0c0", - "reference": "37eec0fe47ddd627911f318f29b6cd48196be0c0", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/0e7804c176c4b09d95b7985400aa38ce544cb7fc", + "reference": "0e7804c176c4b09d95b7985400aa38ce544cb7fc", "shasum": "" }, "require": { @@ -721,7 +721,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-29T21:40:28+00:00" + "time": "2025-04-08T09:55:41+00:00" }, { "name": "open-telemetry/sem-conv", @@ -1771,16 +1771,16 @@ }, { "name": "tbachert/spi", - "version": "v1.0.2", + "version": "v1.0.3", "source": { "type": "git", "url": "https://github.com/Nevay/spi.git", - "reference": "2ddfaf815dafb45791a61b08170de8d583c16062" + "reference": "506a79c98e1a51522e76ee921ccb6c62d52faf3a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Nevay/spi/zipball/2ddfaf815dafb45791a61b08170de8d583c16062", - "reference": "2ddfaf815dafb45791a61b08170de8d583c16062", + "url": "https://api.github.com/repos/Nevay/spi/zipball/506a79c98e1a51522e76ee921ccb6c62d52faf3a", + "reference": "506a79c98e1a51522e76ee921ccb6c62d52faf3a", "shasum": "" }, "require": { @@ -1817,9 +1817,9 @@ ], "support": { "issues": "https://github.com/Nevay/spi/issues", - "source": "https://github.com/Nevay/spi/tree/v1.0.2" + "source": "https://github.com/Nevay/spi/tree/v1.0.3" }, - "time": "2024-10-04T16:36:12+00:00" + "time": "2025-04-02T19:38:14+00:00" }, { "name": "utopia-php/cache", @@ -1920,16 +1920,16 @@ }, { "name": "utopia-php/database", - "version": "0.64.0", + "version": "0.64.2", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "3ec66840dc0bcd24ed1c8e0f6d90fd3a61316ff4" + "reference": "dc9c4a68c93e8bea2dfaa76d1ba308be539998bd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/3ec66840dc0bcd24ed1c8e0f6d90fd3a61316ff4", - "reference": "3ec66840dc0bcd24ed1c8e0f6d90fd3a61316ff4", + "url": "https://api.github.com/repos/utopia-php/database/zipball/dc9c4a68c93e8bea2dfaa76d1ba308be539998bd", + "reference": "dc9c4a68c93e8bea2dfaa76d1ba308be539998bd", "shasum": "" }, "require": { @@ -1970,9 +1970,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.64.0" + "source": "https://github.com/utopia-php/database/tree/0.64.2" }, - "time": "2025-03-28T01:35:56+00:00" + "time": "2025-04-09T07:53:05+00:00" }, { "name": "utopia-php/dsn", @@ -2349,16 +2349,16 @@ }, { "name": "laravel/pint", - "version": "v1.21.2", + "version": "v1.22.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "370772e7d9e9da087678a0edf2b11b6960e40558" + "reference": "7ddfaa6523a675fae5c4123ee38fc6bfb8ee4f36" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/370772e7d9e9da087678a0edf2b11b6960e40558", - "reference": "370772e7d9e9da087678a0edf2b11b6960e40558", + "url": "https://api.github.com/repos/laravel/pint/zipball/7ddfaa6523a675fae5c4123ee38fc6bfb8ee4f36", + "reference": "7ddfaa6523a675fae5c4123ee38fc6bfb8ee4f36", "shasum": "" }, "require": { @@ -2369,9 +2369,9 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.72.0", + "friendsofphp/php-cs-fixer": "^3.75.0", "illuminate/view": "^11.44.2", - "larastan/larastan": "^3.2.0", + "larastan/larastan": "^3.3.1", "laravel-zero/framework": "^11.36.1", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3", @@ -2411,7 +2411,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-03-14T22:31:42+00:00" + "time": "2025-04-08T22:11:45+00:00" }, { "name": "myclabs/deep-copy", @@ -3107,16 +3107,16 @@ }, { "name": "phpunit/phpunit", - "version": "11.5.15", + "version": "11.5.17", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "4b6a4ee654e5e0c5e1f17e2f83c0f4c91dee1f9c" + "reference": "fd2e863a2995cdfd864fb514b5e0b28b09895b5c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/4b6a4ee654e5e0c5e1f17e2f83c0f4c91dee1f9c", - "reference": "4b6a4ee654e5e0c5e1f17e2f83c0f4c91dee1f9c", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/fd2e863a2995cdfd864fb514b5e0b28b09895b5c", + "reference": "fd2e863a2995cdfd864fb514b5e0b28b09895b5c", "shasum": "" }, "require": { @@ -3188,7 +3188,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.15" + "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.17" }, "funding": [ { @@ -3204,7 +3204,7 @@ "type": "tidelift" } ], - "time": "2025-03-23T16:02:11+00:00" + "time": "2025-04-08T07:59:11+00:00" }, { "name": "sebastian/cli-parser", From 52fb030dfb988233dee96845b9c481542fe1a360 Mon Sep 17 00:00:00 2001 From: Darshan Date: Sat, 12 Apr 2025 16:58:18 +0530 Subject: [PATCH 07/14] fix: lint? --- src/Migration/Sources/Csv.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Migration/Sources/Csv.php b/src/Migration/Sources/Csv.php index 789af107..e6dcf7e5 100644 --- a/src/Migration/Sources/Csv.php +++ b/src/Migration/Sources/Csv.php @@ -255,7 +255,7 @@ public function getCollection(string $databaseId, string $collectionId): UtopiaD { $database = $this->dbForProject->getDocument('databases', $databaseId); if ($database->isEmpty()) { - return new UtopiaDocument; + return new UtopiaDocument(); } return $this->dbForProject->getDocument('database_'.$database->getInternalId(), $collectionId); From 59fa4ddd15d6f9b39636cd2773149d27441340be Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 14 Apr 2025 15:37:50 +0530 Subject: [PATCH 08/14] update: change name. --- src/Migration/Sources/Csv.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Migration/Sources/Csv.php b/src/Migration/Sources/Csv.php index e6dcf7e5..fe8d9115 100644 --- a/src/Migration/Sources/Csv.php +++ b/src/Migration/Sources/Csv.php @@ -26,7 +26,7 @@ class Csv extends Source */ private string $resourceId; - private Device $deviceForCsvImports; + private Device $deviceForImports; private Reader $database; @@ -35,12 +35,12 @@ class Csv extends Source public function __construct( string $resourceId, string $filePath, - Device $deviceForCsvImports, + Device $deviceForImports, ?UtopiaDatabase $dbForProject ) { $this->filePath = $filePath; $this->resourceId = $resourceId; - $this->deviceForCsvImports = $deviceForCsvImports; + $this->deviceForImports = $deviceForImports; $this->dbForProject = $dbForProject; $this->database = new DatabaseReader($dbForProject); @@ -89,7 +89,7 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void ); } finally { // delete the temporary file! - $this->deviceForCsvImports->delete($this->filePath); + $this->deviceForImports->delete($this->filePath); } } @@ -98,7 +98,7 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void */ private function exportDocuments(int $batchSize): void { - if (! $this->deviceForCsvImports->exists($this->filePath)) { + if (! $this->deviceForImports->exists($this->filePath)) { return; } From 93cee625d8112d0979ad0315ad3fddcff55f7190 Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 14 Apr 2025 16:11:07 +0530 Subject: [PATCH 09/14] address comments. --- src/Migration/Sources/Csv.php | 40 ++++++----------------------------- 1 file changed, 6 insertions(+), 34 deletions(-) diff --git a/src/Migration/Sources/Csv.php b/src/Migration/Sources/Csv.php index fe8d9115..de2146ee 100644 --- a/src/Migration/Sources/Csv.php +++ b/src/Migration/Sources/Csv.php @@ -3,7 +3,6 @@ namespace Utopia\Migration\Sources; use Utopia\Database\Database as UtopiaDatabase; -use Utopia\Database\Document as UtopiaDocument; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Database\Attribute; @@ -30,8 +29,6 @@ class Csv extends Source private Reader $database; - private ?UtopiaDatabase $dbForProject; - public function __construct( string $resourceId, string $filePath, @@ -41,8 +38,6 @@ public function __construct( $this->filePath = $filePath; $this->resourceId = $resourceId; $this->deviceForImports = $deviceForImports; - - $this->dbForProject = $dbForProject; $this->database = new DatabaseReader($dbForProject); } @@ -113,16 +108,13 @@ private function exportDocuments(int $batchSize): void return; } - $allAttributes = []; + $attributes = []; $lastAttribute = null; [$databaseId, $collectionId] = explode(':', $this->resourceId); $database = new Database($databaseId, ''); $collection = new Collection($database, '', $collectionId); - $collectionStructure = $this->getCollection($databaseId, $collectionId); - $hasDocumentSecurityEnabled = $collectionStructure->getAttribute('documentSecurity', false); - while (true) { $queries = [$this->database->queryLimit($batchSize)]; if ($lastAttribute) { @@ -134,7 +126,7 @@ private function exportDocuments(int $batchSize): void break; } - $allAttributes = array_merge($allAttributes, $fetched); + array_push($attributes, ...$fetched); $lastAttribute = $fetched[count($fetched) - 1]; if (count($fetched) < $batchSize) { @@ -145,7 +137,7 @@ private function exportDocuments(int $batchSize): void $attributeTypes = []; $manyToManyKeys = []; - foreach ($allAttributes as $attribute) { + foreach ($attributes as $attribute) { $key = $attribute['key']; if ( @@ -186,8 +178,6 @@ private function exportDocuments(int $batchSize): void $parsedValue = trim($value); if ($parsedValue === '') { - $parsedData[$key] = null; - continue; } @@ -220,15 +210,12 @@ private function exportDocuments(int $batchSize): void $permissions = []; $documentId = $parsedData['$id'] ?? 'unique()'; - if ($hasDocumentSecurityEnabled && isset($parsedData['$permissions'])) { + if (isset($parsedData['$permissions'])) { $permissions = $this->parsePermissions($parsedData['$permissions']); } - foreach ($parsedData as $key => $value) { - if (str_starts_with($key, '$')) { - unset($parsedData[$key]); - } - } + unset($parsedData['$id']); + unset($parsedData['$permissions']); $document = new Document($documentId, $collection, $parsedData, $permissions); $buffer[] = $document; @@ -246,21 +233,6 @@ private function exportDocuments(int $batchSize): void } } - /** - * Fast path function without the built-in `listCollections` for better performance! - * - * @throws \Exception - */ - public function getCollection(string $databaseId, string $collectionId): UtopiaDocument - { - $database = $this->dbForProject->getDocument('databases', $databaseId); - if ($database->isEmpty()) { - return new UtopiaDocument(); - } - - return $this->dbForProject->getDocument('database_'.$database->getInternalId(), $collectionId); - } - /** * Parses a stringified permission array into a string[]. * From d1c6bc0179ed7686ede2577e2fe41f4082aaabd2 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 16 Apr 2025 11:31:48 +0530 Subject: [PATCH 10/14] address comments. --- src/Migration/Sources/{Csv.php => CSV.php} | 191 +++++++++------------ 1 file changed, 81 insertions(+), 110 deletions(-) rename src/Migration/Sources/{Csv.php => CSV.php} (60%) diff --git a/src/Migration/Sources/Csv.php b/src/Migration/Sources/CSV.php similarity index 60% rename from src/Migration/Sources/Csv.php rename to src/Migration/Sources/CSV.php index de2146ee..66144db0 100644 --- a/src/Migration/Sources/Csv.php +++ b/src/Migration/Sources/CSV.php @@ -16,7 +16,7 @@ use Utopia\Migration\Transfer; use Utopia\Storage\Device; -class Csv extends Source +class CSV extends Source { private string $filePath; @@ -25,25 +25,25 @@ class Csv extends Source */ private string $resourceId; - private Device $deviceForImports; + private Device $device; private Reader $database; public function __construct( string $resourceId, string $filePath, - Device $deviceForImports, + Device $device, ?UtopiaDatabase $dbForProject ) { + $this->device = $device; $this->filePath = $filePath; $this->resourceId = $resourceId; - $this->deviceForImports = $deviceForImports; $this->database = new DatabaseReader($dbForProject); } public static function getName(): string { - return 'Csv'; + return 'CSV'; } public static function getSupportedResources(): array @@ -53,9 +53,26 @@ public static function getSupportedResources(): array ]; } + // called before the `exportGroupDatabases`. public function report(array $resources = []): array { - return []; + $report = []; + + $this->withCsvStream(function ($stream) use (&$report) { + $headers = fgetcsv($stream); + if (! is_array($headers) || count($headers) === 0) { + return; + } + + $rowCount = 0; + while (fgetcsv($stream) !== false) { + $rowCount++; + } + + $report[Resource::TYPE_DOCUMENT] = $rowCount; + }); + + return $report; } /** @@ -84,7 +101,7 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void ); } finally { // delete the temporary file! - $this->deviceForImports->delete($this->filePath); + $this->device->delete($this->filePath); } } @@ -93,20 +110,6 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void */ private function exportDocuments(int $batchSize): void { - if (! $this->deviceForImports->exists($this->filePath)) { - return; - } - - $stream = fopen($this->filePath, 'r'); - if (! $stream) { - return; - } - - $headers = fgetcsv($stream); - if (! is_array($headers) || count($headers) === 0) { - fclose($stream); - return; - } $attributes = []; $lastAttribute = null; @@ -141,7 +144,6 @@ private function exportDocuments(int $batchSize): void $key = $attribute['key']; if ( - // Skip child-side relationships entirely $attribute['type'] === Attribute::TYPE_RELATIONSHIP && ($attribute['side'] ?? '') === UtopiaDatabase::RELATION_SIDE_CHILD ) { @@ -159,113 +161,64 @@ private function exportDocuments(int $batchSize): void } } - $buffer = []; - - while (($row = fgetcsv($stream)) !== false) { - $data = array_combine($headers, $row); - if ($data === false) { - continue; + $this->withCSVStream(function ($stream) use ($attributeTypes, $manyToManyKeys, $collection, $batchSize) { + $headers = fgetcsv($stream); + if (! is_array($headers) || count($headers) === 0) { + return; } - $parsedData = $data; - - foreach ($data as $key => $value) { - if (! isset($attributeTypes[$key])) { - continue; - } - - $type = $attributeTypes[$key]; - $parsedValue = trim($value); + $buffer = []; - if ($parsedValue === '') { + while (($row = fgetcsv($stream)) !== false) { + $data = array_combine($headers, $row); + if ($data === false) { continue; } - if (in_array($key, $manyToManyKeys, true)) { - $parsedData[$key] = str_contains($parsedValue, ',') - ? array_map('trim', explode(',', $parsedValue)) - : [$parsedValue]; + $parsedData = $data; - continue; - } + foreach ($data as $key => $value) { + $parsedValue = trim($value); + $type = $attributeTypes[$key]; - switch ($type) { - case Attribute::TYPE_INTEGER: - $parsedData[$key] = is_numeric($parsedValue) ? (int) $parsedValue : null; - break; + if (! isset($type) || $parsedValue === '') { + continue; + } - case Attribute::TYPE_FLOAT: - $parsedData[$key] = is_numeric($parsedValue) ? (float) $parsedValue : null; - break; + if (in_array($key, $manyToManyKeys, true)) { + $parsedData[$key] = str_contains($parsedValue, ',') + ? array_map('trim', explode(',', $parsedValue)) + : [$parsedValue]; - case Attribute::TYPE_BOOLEAN: - $parsedData[$key] = filter_var($parsedValue, FILTER_VALIDATE_BOOLEAN); - break; + continue; + } - default: - break; + $parsedData[$key] = match ($type) { + Attribute::TYPE_INTEGER => is_numeric($parsedValue) ? (int) $parsedValue : null, + Attribute::TYPE_FLOAT => is_numeric($parsedValue) ? (float) $parsedValue : null, + Attribute::TYPE_BOOLEAN => filter_var($parsedValue, FILTER_VALIDATE_BOOLEAN), + default => $parsedValue, + }; } - } - $permissions = []; - $documentId = $parsedData['$id'] ?? 'unique()'; + $documentId = $parsedData['$id'] ?? 'unique()'; - if (isset($parsedData['$permissions'])) { - $permissions = $this->parsePermissions($parsedData['$permissions']); - } + // `$id`, `$permissions` in the doc can cause issues! + unset($parsedData['$id'], $parsedData['$permissions']); - unset($parsedData['$id']); - unset($parsedData['$permissions']); + $document = new Document($documentId, $collection, $parsedData); + $buffer[] = $document; - $document = new Document($documentId, $collection, $parsedData, $permissions); - $buffer[] = $document; + if (count($buffer) === $batchSize) { + $this->callback($buffer); + $buffer = []; + } + } - if (count($buffer) === $batchSize) { + if (! empty($buffer)) { $this->callback($buffer); - $buffer = []; } - } - - fclose($stream); - - if (! empty($buffer)) { - $this->callback($buffer); - } - } - - /** - * Parses a stringified permission array into a string[]. - * - * Example: - * ``` - * "[read(\"user:user1234\"),read(\"user:user4321\")]" - * ``` - * Into: - * ``` - * [ - * "read(\"user:user1234\")", - * "read(\"user:user4321\")" - * ] - * ``` - * - * @param string $raw - * @return string[] - */ - private function parsePermissions(string $raw): array - { - $raw = trim($raw, ' "[]'); - - if (empty($raw)) { - return []; - } - - $parts = preg_split('/,(?![^(]*\))/', $raw); - - return array_map(function ($item) { - $item = trim($item); - - return str_replace('\"', '"', $item); - }, $parts); + }); } /** @@ -307,4 +260,22 @@ protected function exportGroupFunctions(int $batchSize, array $resources): void { throw new \Exception('Not Implemented'); } + + private function withCsvStream(callable $fn): void + { + if (! $this->device->exists($this->filePath)) { + return; + } + + $stream = fopen($this->filePath, 'r'); + if (! $stream) { + return; + } + + try { + $fn($stream); + } finally { + fclose($stream); + } + } } From 9088ef1079da3fb13b8abc3821feee618475a0c3 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 16 Apr 2025 11:51:15 +0530 Subject: [PATCH 11/14] fix: `Undefined array key`. --- src/Migration/Sources/CSV.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index 66144db0..889c4410 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -179,7 +179,7 @@ private function exportDocuments(int $batchSize): void foreach ($data as $key => $value) { $parsedValue = trim($value); - $type = $attributeTypes[$key]; + $type = $attributeTypes[$key] ?? null; if (! isset($type) || $parsedValue === '') { continue; From 7357615537163a5199cedee143c611c419e03921 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 16 Apr 2025 12:40:16 +0530 Subject: [PATCH 12/14] address comment: use `SPLFileObject`. --- src/Migration/Sources/CSV.php | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index 889c4410..5718595b 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -58,19 +58,30 @@ public function report(array $resources = []): array { $report = []; - $this->withCsvStream(function ($stream) use (&$report) { - $headers = fgetcsv($stream); - if (! is_array($headers) || count($headers) === 0) { - return; - } + if (! $this->device->exists($this->filePath)) { + return $report; + } + + $file = new \SplFileObject($this->filePath, 'r'); + $file->setFlags(\SplFileObject::READ_CSV | \SplFileObject::SKIP_EMPTY); - $rowCount = 0; - while (fgetcsv($stream) !== false) { - $rowCount++; + if (! $file->eof()) { + $file->fgetcsv(); + } + + $rowCount = 0; + while (! $file->eof()) { + $row = $file->fgetcsv(); + + // check for blank lines + if ($row === [null] || $row === false) { + continue; } - $report[Resource::TYPE_DOCUMENT] = $rowCount; - }); + $rowCount++; + } + + $report[Resource::TYPE_DOCUMENT] = $rowCount; return $report; } From ff392fe80415df6fb49ff72c900d1997b917fbd5 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 16 Apr 2025 12:49:42 +0530 Subject: [PATCH 13/14] address comment: use `SPLFileObject` with `seek`. --- src/Migration/Sources/CSV.php | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index 5718595b..b48f5536 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -65,20 +65,12 @@ public function report(array $resources = []): array $file = new \SplFileObject($this->filePath, 'r'); $file->setFlags(\SplFileObject::READ_CSV | \SplFileObject::SKIP_EMPTY); - if (! $file->eof()) { - $file->fgetcsv(); - } - - $rowCount = 0; - while (! $file->eof()) { - $row = $file->fgetcsv(); - - // check for blank lines - if ($row === [null] || $row === false) { - continue; - } + $file->seek(PHP_INT_MAX); + $rowCount = $file->key(); - $rowCount++; + // Subtract to exclude header + if ($rowCount > 0) { + $rowCount--; } $report[Resource::TYPE_DOCUMENT] = $rowCount; From f321d089a2703ccaec5f259aaba4d0f19c9a00f7 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 16 Apr 2025 13:17:17 +0530 Subject: [PATCH 14/14] update: `$rowCount`. --- src/Migration/Sources/CSV.php | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index b48f5536..fed64d74 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -66,12 +66,8 @@ public function report(array $resources = []): array $file->setFlags(\SplFileObject::READ_CSV | \SplFileObject::SKIP_EMPTY); $file->seek(PHP_INT_MAX); - $rowCount = $file->key(); - - // Subtract to exclude header - if ($rowCount > 0) { - $rowCount--; - } + $rowCount = max(0, $file->key()); + $rowCount = $rowCount > 0 ? $rowCount - 1 : 0; $report[Resource::TYPE_DOCUMENT] = $rowCount;