feat(UserMigration): Overwork migration to include all settings (user-specific app configuration)

.github/workflows/test.yml

@@ -304,13 +304,65 @@ jobs:
     env:
       CI: true
 
+  user-migration-smoke-test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        php-versions: [ '8.4' ]
+        nextcloud-versions: [ 'stable33' ]
+        db: [ 'mysql' ]
+    env:
+      TEST_USER: user1
+      EXPORT_DIRECTORY: /tmp
+    name: Nextcloud ${{ matrix.nextcloud-versions }} php${{ matrix.php-versions }} user migration test
+    steps:
+      - name: Set up Nextcloud env
+        uses: nextcloud/setup-server-action@34b73d5b0e3633f83a52227d00cc2a6c41d01d9a # v1.0.0
+        with:
+          nextcloud-version: ${{ matrix.nextcloud-versions }}
+          php-version: ${{ matrix.php-versions }}
+          patch-php-version-check: ${{ matrix.php-versions == '8.6' }}
+          node-version: 'false'
+          install: true
+      - name: Checkout mail
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          path: nextcloud/apps/mail
+          fetch-depth: 2
+      - name: Install dependencies
+        working-directory: nextcloud/apps/mail
+        run: composer install --no-dev
+      - name: Install mail
+        run: php -f nextcloud/occ app:enable mail
+      - name: Checkout user_migration
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          repository: nextcloud/user_migration
+          path: nextcloud/apps/user_migration
+      - name: Install dependencies
+        working-directory: nextcloud/apps/user_migration
+        run: composer install --no-dev
+      - name: Install user_migration
+        run: php -f nextcloud/occ app:enable user_migration
+      - name: Create test user
+        run: php -f nextcloud/occ user:add --generate-password $TEST_USER
+      - name: Test basic mail export
+        run: php -f nextcloud/occ user:export -t mail_account $TEST_USER $EXPORT_DIRECTORY
+      - name: Delete exported user to avoid overwrite confirmation during import
+        run: php -f nextcloud/occ user:delete $TEST_USER
+      - name: Test basic mail import
+        run: |
+          EXPORTED_ZIP=$(find $EXPORT_DIRECTORY -name "$TEST_USER*.zip" | tail -1)
+          php -f nextcloud/occ user:import $EXPORTED_ZIP
+
   summary:
     runs-on: ubuntu-latest-low
     needs:
       - unit-tests
       - integration-tests
       - frontend-unit-test
       - frontend-e2e-tests
+      - user-migration-smoke-test
 
     if: always()
 
@@ -325,3 +377,5 @@ jobs:
         run: if ${{ needs.frontend-unit-test.result != 'success' && needs.frontend-unit-test.result != 'skipped' }}; then exit 1; fi
       - name: Frontend E2E test status
         run: if ${{ needs.frontend-e2e-tests.result != 'success' && needs.frontend-e2e-tests.result != 'skipped' }}; then exit 1; fi
+      - name: User migration smoke test status
+        run: if ${{ needs.user-migration-smoke-test.result != 'success' && needs.user-migration-smoke-test.result != 'skipped' }}; then exit 1; fi

lib/Service/AccountService.php

@@ -149,6 +149,10 @@ public function delete(string $currentUserId, int $accountId): void {
 		}
 		$this->aliasesService->deleteAll($accountId);
 		$this->mapper->delete($mailAccount);
+
+		// Invalidate cache to ensure deleted account is not included
+		// in subsequent `findByUserId` and `findByUserIdAndAddress` calls
+		unset($this->accounts[$currentUserId]);
 	}
 
 	/**
@@ -164,17 +168,31 @@ public function deleteByAccountId(int $accountId): void {
 		}
 		$this->aliasesService->deleteAll($accountId);
 		$this->mapper->delete($mailAccount);
+
+		// Invalidate cache to ensure deleted account is not included
+		// in subsequent `findByUserId` and `findByUserIdAndAddress` calls
+		unset($this->accounts[$mailAccount->getUserId()]);
 	}
 
 	/**
 	 * @param MailAccount $newAccount
+	 * @param bool $scheduleBackgroundJobs Optional parameter to save the mail account
+	 *                                     without scheduling the corresponding background jobs. This can be useful if
+	 *                                     further database modifications must be done before running any background
+	 *                                     jobs. Defaults to `true`.
 	 * @return MailAccount
 	 */
-	public function save(MailAccount $newAccount): MailAccount {
+	public function save(MailAccount $newAccount, bool $scheduleBackgroundJobs = true): MailAccount {
 		$newAccount = $this->mapper->save($newAccount);
 
 		// Insert background jobs for this account
-		$this->scheduleBackgroundJobs($newAccount->getId());
+		if ($scheduleBackgroundJobs) {
+			$this->scheduleBackgroundJobs($newAccount->getId());
+		}
+
+		// Invalidate cache to ensure created account is being included
+		// in subsequent `findByUserId` and `findByUserIdAndAddress` calls
+		unset($this->accounts[$newAccount->getUserId()]);
 
 		return $newAccount;
 	}
@@ -196,6 +214,10 @@ public function updateSignature(int $id, string $uid, ?string $signature = null)
 		$mailAccount = $account->getMailAccount();
 		$mailAccount->setSignature($signature);
 		$this->mapper->save($mailAccount);
+
+		// Invalidate cache to ensure changed signature is being included
+		// in subsequent `findByUserId` and `findByUserIdAndAddress` calls
+		unset($this->accounts[$uid]);
 	}
 
 	/**

lib/UserMigration/MailAccountMigrator.php

@@ -4,17 +4,17 @@
 
 /**
  * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
- * SPDX-License-Identifier: AGPL-3.0-only
+ * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
 namespace OCA\Mail\UserMigration;
 
-use Exception;
-use JsonException;
-use OCA\Mail\Db\Alias;
-use OCA\Mail\Db\MailAccount;
-use OCA\Mail\Service\AccountService;
-use OCA\Mail\Service\AliasesService;
+use OCA\Mail\AppInfo\Application;
+use OCA\Mail\Exception\ClientException;
+use OCA\Mail\Exception\ServiceException;
+use OCA\Mail\UserMigration\Service\AccountMigrationService;
+use OCA\Mail\UserMigration\Service\AppConfigMigrationService;
+use OCP\AppFramework\Db\DoesNotExistException;
 use OCP\IL10N;
 use OCP\IUser;
 use OCP\Security\ICrypto;
@@ -23,17 +23,16 @@
 use OCP\UserMigration\IMigrator;
 use OCP\UserMigration\UserMigrationException;
 use Symfony\Component\Console\Output\OutputInterface;
-use function array_map;
-use function json_decode;
-use function json_encode;
 
 class MailAccountMigrator implements IMigrator {
+	public const EXPORT_ROOT = Application::APP_ID;
+	public const FILENAME_PLACEHOLDER = '{filename}';
 
 	public function __construct(
-		private AccountService $accountService,
-		private AliasesService $aliasesService,
-		private IL10N $l10n,
-		private ICrypto $crypto,
+		private readonly IL10N $l10n,
+		private readonly ICrypto $crypto,
+		private readonly AccountMigrationService $accountMigrationService,
+		private readonly AppConfigMigrationService $appConfigMigrationService,
 	) {
 	}
 
@@ -42,140 +41,36 @@ public function export(IUser $user,
 		IExportDestination $exportDestination,
 		OutputInterface $output,
 	): void {
-		$accounts = $this->accountService->findByUserId($user->getUID());
-		$index = [];
-		foreach ($accounts as $account) {
-			if ($account->getMailAccount()->getProvisioningId() !== null) {
-				// These configuration of these accounts is owned by the admins
-				$output->writeln("Skipping provisioned account {$account->getId()}");
-				continue;
-			}
-
-			$accountFilePath = "mail/accounts/{$account->getId()}.json";
-			$accountData = $account->jsonSerialize();
-
-			if ($account->getMailAccount()->getAuthMethod() === 'password') {
-				$encryptedInboundPassword = $account->getMailAccount()->getInboundPassword();
-				$encryptedOutboundPassword = $account->getMailAccount()->getOutboundPassword();
-				if ($encryptedInboundPassword !== null) {
-					try {
-						$accountData['inboundPassword'] = $this->crypto->decrypt($encryptedInboundPassword);
-					} catch (Exception $e) {
-						$output->writeln("Can not decrypt inbound password of account {$account->getId()}: " . $e->getMessage());
-					}
-				}
-				if ($encryptedOutboundPassword !== null) {
-					try {
-						$accountData['outboundPassword'] = $this->crypto->decrypt($encryptedOutboundPassword);
-					} catch (Exception $e) {
-						$output->writeln("Can not decrypt outbound password of account {$account->getId()}: " . $e->getMessage());
-					}
-				}
-			} elseif ($account->getMailAccount()->getAuthMethod() === 'xoauth2') {
-				$encryptedRefreshToken = $account->getMailAccount()->getOauthRefreshToken();
-				$encryptedAccessToken = $account->getMailAccount()->getOauthAccessToken();
-				if ($encryptedRefreshToken !== null) {
-					try {
-						$accountData['oauthRefreshToken'] = $this->crypto->decrypt($encryptedRefreshToken);
-					} catch (Exception $e) {
-						$output->writeln("Can not decrypt oauth refresh token of account {$account->getId()}: " . $e->getMessage());
-					}
-				}
-				if ($encryptedAccessToken !== null) {
-					try {
-						$accountData['oauthAccessToken'] = $this->crypto->decrypt($encryptedAccessToken);
-					} catch (Exception $e) {
-						$output->writeln("Can not decrypt oauth access token of account {$account->getId()}: " . $e->getMessage());
-					}
-				}
-				$accountData['oauthTokenTtl'] = $account->getMailAccount()->getOauthTokenTtl();
-			}
-
-			unset(
-				$accountData['draftsMailboxId'],
-				$accountData['sentMailboxId'],
-				$accountData['trashMailboxId'],
-				$accountData['archiveMailboxId'],
-				$accountData['snoozeMailboxId'],
-				$accountData['junkMailboxId'],
-			);
-
-			$aliases = $this->aliasesService->findAll(
-				$account->getId(),
-				$account->getUserId(), // perf: this adds overhead - add dedicated method to fetch by account id only
-			);
-			$accountData['aliases'] = array_map(function (Alias $alias) {
-				$data = $alias->jsonSerialize();
-				return $data;
-			}, $aliases);
-
-			$exportDestination->addFileContents($accountFilePath, json_encode($accountData));
-			$index[$account->getId()] = $accountFilePath;
-		}
+		$output->writeln($this->l10n->t("Exporting mail accounts for user {$user->getUID()}"), OutputInterface::VERBOSITY_VERBOSE);
 
-		$exportDestination->addFileContents('mail/accounts/index.json', json_encode($index));
+		$this->appConfigMigrationService->exportAppConfiguration($user, $exportDestination, $output);
 	}
 
 	#[\Override]
 	public function import(IUser $user, IImportSource $importSource, OutputInterface $output): void {
-		try {
-			$index = json_decode($importSource->getFileContents('mail/accounts/index.json'), true, flags: JSON_THROW_ON_ERROR);
-		} catch (JsonException $e) {
-			throw new UserMigrationException("Invalid index content: {$e->getMessage()}", $e->getCode(), $e);
-		}
-		foreach ($index as $accountFilePath) {
-			try {
-				$accountData = json_decode($importSource->getFileContents($accountFilePath), true, flags: JSON_THROW_ON_ERROR);
-			} catch (JsonException $e) {
-				throw new UserMigrationException("Invalid account content: {$e->getMessage()}", $e->getCode(), $e);
-			}
-
-			// Wipe the old ID(s) to prevent overwrites
-			unset(
-				$accountData['id'],
-				$accountData['accountId'],
-			);
-
-			$newAccount = new MailAccount($accountData);
-
-			// Change UID to new owner
-			$newAccount->setUserId($user->getUID());
-			// Map the rest of the properties that are not mapped via the constructor
-			$newAccount->setName($accountData['name']);
-			$newAccount->setAuthMethod($accountData['authMethod']);
-			$newAccount->setEditorMode($accountData['editorMode'] ?? 'plain');
-			$newAccount->setSearchBody($accountData['searchBody'] ?? false);
-			$newAccount->setClassificationEnabled($accountData['classificationEnabled'] ?? false);
-			$newAccount->setSignatureAboveQuote($accountData['signatureAboveQuote'] ?? false);
-			$newAccount->setPersonalNamespace($accountData['personalNamespace'] ?? null);
-			if (isset($accountData['inboundPassword'])) {
-				$newAccount->setInboundPassword($this->crypto->encrypt($accountData['inboundPassword']));
-			}
-			if (isset($accountData['outboundPassword'])) {
-				$newAccount->setOutboundPassword($this->crypto->encrypt($accountData['outboundPassword']));
-			}
-			if (isset($accountData['oauthRefreshToken'])) {
-				$newAccount->setOauthRefreshToken($this->crypto->encrypt($accountData['oauthRefreshToken']));
-			}
-			if (isset($accountData['oauthAccessToken'])) {
-				$newAccount->setOauthAccessToken($this->crypto->encrypt($accountData['oauthAccessToken']));
-			}
-			$newAccount->setOauthTokenTtl($accountData['oauthTokenTtl'] ?? null);
-
-			$mailAccount = $this->accountService->save(
-				$newAccount
-			);
-
-			// Import aliases
-			foreach ($accountData['aliases'] as $alias) {
-				$this->aliasesService->create(
-					$user->getUID(),
-					$mailAccount->getId(),
-					$alias['alias'],
-					$alias['name'],
-				);
-			}
-		}
+		$output->writeln($this->l10n->t("Importing mail accounts for user {$user->getUID()}"), OutputInterface::VERBOSITY_VERBOSE);
+		$this->deleteExistingData($user, $output);
+
+		$this->appConfigMigrationService->importAppConfiguration($user, $importSource, $output);
+
+		$this->accountMigrationService->scheduleBackgroundJobs($user, $output);
+	}
+
+	/**
+	 * Delete all existing user data of our app to ensure
+	 * the result of the import is always the same.
+	 *
+	 * @param IUser $user
+	 * @param OutputInterface $output
+	 * @throws ClientException
+	 * @throws DoesNotExistException
+	 * @throws ServiceException
+	 */
+	private function deleteExistingData(IUser $user, OutputInterface $output): void {
+		$output->writeln($this->l10n->t("Deleting existing mail data for user {$user->getUID()}"), OutputInterface::VERBOSITY_VERBOSE);
+
+		$this->appConfigMigrationService->deleteAppConfiguration($user, $output);
+		$this->accountMigrationService->deleteAllAccounts($user, $output);
 	}
 
 	#[\Override]
@@ -195,7 +90,7 @@ public function getDescription(): string {
 
 	#[\Override]
 	public function getVersion(): int {
-		return 01_00_00;
+		return 02_00_00;
 	}
 
 	#[\Override]