Skip to content

Commit a56351e

Browse files
DerDreschnerDerDreschner
committed
feat(UserMigration): Overwork migration to include all settings (basic changes)
Signed-off-by: David Dreschner <david.dreschner@nextcloud.com>
1 parent fffd17f commit a56351e

7 files changed

Lines changed: 260 additions & 317 deletions

File tree

‎.github/workflows/test.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,13 +304,65 @@ jobs:
304304
env:
305305
CI: true
306306

307+
user-migration-smoke-test:
308+
runs-on: ubuntu-latest
309+
strategy:
310+
matrix:
311+
php-versions: [ '8.4' ]
312+
nextcloud-versions: [ 'stable33' ]
313+
db: [ 'mysql' ]
314+
env:
315+
TEST_USER: user1
316+
EXPORT_DIRECTORY: /tmp
317+
name: Nextcloud ${{ matrix.nextcloud-versions }} php${{ matrix.php-versions }} user migration test
318+
steps:
319+
- name: Set up Nextcloud env
320+
uses: nextcloud/setup-server-action@34b73d5b0e3633f83a52227d00cc2a6c41d01d9a # v1.0.0
321+
with:
322+
nextcloud-version: ${{ matrix.nextcloud-versions }}
323+
php-version: ${{ matrix.php-versions }}
324+
patch-php-version-check: ${{ matrix.php-versions == '8.6' }}
325+
node-version: 'false'
326+
install: true
327+
- name: Checkout mail
328+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
329+
with:
330+
path: nextcloud/apps/mail
331+
fetch-depth: 2
332+
- name: Install dependencies
333+
working-directory: nextcloud/apps/mail
334+
run: composer install --no-dev
335+
- name: Install mail
336+
run: php -f nextcloud/occ app:enable mail
337+
- name: Checkout user_migration
338+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
339+
with:
340+
repository: nextcloud/user_migration
341+
path: nextcloud/apps/user_migration
342+
- name: Install dependencies
343+
working-directory: nextcloud/apps/user_migration
344+
run: composer install --no-dev
345+
- name: Install user_migration
346+
run: php -f nextcloud/occ app:enable user_migration
347+
- name: Create test user
348+
run: php -f nextcloud/occ user:add --generate-password $TEST_USER
349+
- name: Test basic mail export
350+
run: php -f nextcloud/occ user:export -t mail_account $TEST_USER $EXPORT_DIRECTORY
351+
- name: Delete exported user to avoid overwrite confirmation during import
352+
run: php -f nextcloud/occ user:delete $TEST_USER
353+
- name: Test basic mail import
354+
run: |
355+
EXPORTED_ZIP=$(find $EXPORT_DIRECTORY -name "$TEST_USER*.zip" | tail -1)
356+
php -f nextcloud/occ user:import $EXPORTED_ZIP
357+
307358
summary:
308359
runs-on: ubuntu-latest-low
309360
needs:
310361
- unit-tests
311362
- integration-tests
312363
- frontend-unit-test
313364
- frontend-e2e-tests
365+
- user-migration-smoke-test
314366

315367
if: always()
316368

@@ -325,3 +377,5 @@ jobs:
325377
run: if ${{ needs.frontend-unit-test.result != 'success' && needs.frontend-unit-test.result != 'skipped' }}; then exit 1; fi
326378
- name: Frontend E2E test status
327379
run: if ${{ needs.frontend-e2e-tests.result != 'success' && needs.frontend-e2e-tests.result != 'skipped' }}; then exit 1; fi
380+
- name: User migration smoke test status
381+
run: if ${{ needs.user-migration-smoke-test.result != 'success' && needs.user-migration-smoke-test.result != 'skipped' }}; then exit 1; fi

‎lib/Service/AccountService.php‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,13 +168,19 @@ public function deleteByAccountId(int $accountId): void {
168168

169169
/**
170170
* @param MailAccount $newAccount
171+
* @param bool $scheduleBackgroundJobs Optional parameter to save the mail account
172+
* without scheduling the corresponding background jobs. This can be useful if
173+
* further database modifications must be done before running any background
174+
* jobs. Defaults to `true`.
171175
* @return MailAccount
172176
*/
173-
public function save(MailAccount $newAccount): MailAccount {
177+
public function save(MailAccount $newAccount, bool $scheduleBackgroundJobs = true): MailAccount {
174178
$newAccount = $this->mapper->save($newAccount);
175179

176180
// Insert background jobs for this account
177-
$this->scheduleBackgroundJobs($newAccount->getId());
181+
if ($scheduleBackgroundJobs) {
182+
$this->scheduleBackgroundJobs($newAccount->getId());
183+
}
178184

179185
return $newAccount;
180186
}

‎lib/UserMigration/MailAccountMigrator.php‎

Lines changed: 31 additions & 145 deletions
Original file line numberDiff line numberDiff line change
@@ -9,31 +9,26 @@
99

1010
namespace OCA\Mail\UserMigration;
1111

12-
use Exception;
13-
use JsonException;
14-
use OCA\Mail\Db\Alias;
15-
use OCA\Mail\Db\MailAccount;
16-
use OCA\Mail\Service\AccountService;
17-
use OCA\Mail\Service\AliasesService;
12+
use OCA\Mail\AppInfo\Application;
13+
use OCA\Mail\Exception\ClientException;
14+
use OCA\Mail\Exception\ServiceException;
15+
use OCA\Mail\UserMigration\Service\AccountMigrationService;
16+
use OCP\AppFramework\Db\DoesNotExistException;
1817
use OCP\IL10N;
1918
use OCP\IUser;
20-
use OCP\Security\ICrypto;
2119
use OCP\UserMigration\IExportDestination;
2220
use OCP\UserMigration\IImportSource;
2321
use OCP\UserMigration\IMigrator;
2422
use OCP\UserMigration\UserMigrationException;
2523
use Symfony\Component\Console\Output\OutputInterface;
26-
use function array_map;
27-
use function json_decode;
28-
use function json_encode;
2924

3025
class MailAccountMigrator implements IMigrator {
26+
public const EXPORT_ROOT = Application::APP_ID;
27+
public const FILENAME_PLACEHOLDER = '{filename}';
3128

3229
public function __construct(
33-
private AccountService $accountService,
34-
private AliasesService $aliasesService,
35-
private IL10N $l10n,
36-
private ICrypto $crypto,
30+
private readonly IL10N $l10n,
31+
private readonly AccountMigrationService $accountMigrationService,
3732
) {
3833
}
3934

@@ -42,140 +37,31 @@ public function export(IUser $user,
4237
IExportDestination $exportDestination,
4338
OutputInterface $output,
4439
): void {
45-
$accounts = $this->accountService->findByUserId($user->getUID());
46-
$index = [];
47-
foreach ($accounts as $account) {
48-
if ($account->getMailAccount()->getProvisioningId() !== null) {
49-
// These configuration of these accounts is owned by the admins
50-
$output->writeln("Skipping provisioned account {$account->getId()}");
51-
continue;
52-
}
53-
54-
$accountFilePath = "mail/accounts/{$account->getId()}.json";
55-
$accountData = $account->jsonSerialize();
56-
57-
if ($account->getMailAccount()->getAuthMethod() === 'password') {
58-
$encryptedInboundPassword = $account->getMailAccount()->getInboundPassword();
59-
$encryptedOutboundPassword = $account->getMailAccount()->getOutboundPassword();
60-
if ($encryptedInboundPassword !== null) {
61-
try {
62-
$accountData['inboundPassword'] = $this->crypto->decrypt($encryptedInboundPassword);
63-
} catch (Exception $e) {
64-
$output->writeln("Can not decrypt inbound password of account {$account->getId()}: " . $e->getMessage());
65-
}
66-
}
67-
if ($encryptedOutboundPassword !== null) {
68-
try {
69-
$accountData['outboundPassword'] = $this->crypto->decrypt($encryptedOutboundPassword);
70-
} catch (Exception $e) {
71-
$output->writeln("Can not decrypt outbound password of account {$account->getId()}: " . $e->getMessage());
72-
}
73-
}
74-
} elseif ($account->getMailAccount()->getAuthMethod() === 'xoauth2') {
75-
$encryptedRefreshToken = $account->getMailAccount()->getOauthRefreshToken();
76-
$encryptedAccessToken = $account->getMailAccount()->getOauthAccessToken();
77-
if ($encryptedRefreshToken !== null) {
78-
try {
79-
$accountData['oauthRefreshToken'] = $this->crypto->decrypt($encryptedRefreshToken);
80-
} catch (Exception $e) {
81-
$output->writeln("Can not decrypt oauth refresh token of account {$account->getId()}: " . $e->getMessage());
82-
}
83-
}
84-
if ($encryptedAccessToken !== null) {
85-
try {
86-
$accountData['oauthAccessToken'] = $this->crypto->decrypt($encryptedAccessToken);
87-
} catch (Exception $e) {
88-
$output->writeln("Can not decrypt oauth access token of account {$account->getId()}: " . $e->getMessage());
89-
}
90-
}
91-
$accountData['oauthTokenTtl'] = $account->getMailAccount()->getOauthTokenTtl();
92-
}
93-
94-
unset(
95-
$accountData['draftsMailboxId'],
96-
$accountData['sentMailboxId'],
97-
$accountData['trashMailboxId'],
98-
$accountData['archiveMailboxId'],
99-
$accountData['snoozeMailboxId'],
100-
$accountData['junkMailboxId'],
101-
);
102-
103-
$aliases = $this->aliasesService->findAll(
104-
$account->getId(),
105-
$account->getUserId(), // perf: this adds overhead - add dedicated method to fetch by account id only
106-
);
107-
$accountData['aliases'] = array_map(function (Alias $alias) {
108-
$data = $alias->jsonSerialize();
109-
return $data;
110-
}, $aliases);
111-
112-
$exportDestination->addFileContents($accountFilePath, json_encode($accountData));
113-
$index[$account->getId()] = $accountFilePath;
114-
}
115-
116-
$exportDestination->addFileContents('mail/accounts/index.json', json_encode($index));
40+
$output->writeln("Exporting mail accounts for user {$user->getUID()}", OutputInterface::VERBOSITY_VERBOSE);
11741
}
11842

11943
#[\Override]
12044
public function import(IUser $user, IImportSource $importSource, OutputInterface $output): void {
121-
try {
122-
$index = json_decode($importSource->getFileContents('mail/accounts/index.json'), true, flags: JSON_THROW_ON_ERROR);
123-
} catch (JsonException $e) {
124-
throw new UserMigrationException("Invalid index content: {$e->getMessage()}", $e->getCode(), $e);
125-
}
126-
foreach ($index as $accountFilePath) {
127-
try {
128-
$accountData = json_decode($importSource->getFileContents($accountFilePath), true, flags: JSON_THROW_ON_ERROR);
129-
} catch (JsonException $e) {
130-
throw new UserMigrationException("Invalid account content: {$e->getMessage()}", $e->getCode(), $e);
131-
}
132-
133-
// Wipe the old ID(s) to prevent overwrites
134-
unset(
135-
$accountData['id'],
136-
$accountData['accountId'],
137-
);
138-
139-
$newAccount = new MailAccount($accountData);
140-
141-
// Change UID to new owner
142-
$newAccount->setUserId($user->getUID());
143-
// Map the rest of the properties that are not mapped via the constructor
144-
$newAccount->setName($accountData['name']);
145-
$newAccount->setAuthMethod($accountData['authMethod']);
146-
$newAccount->setEditorMode($accountData['editorMode'] ?? 'plain');
147-
$newAccount->setSearchBody($accountData['searchBody'] ?? false);
148-
$newAccount->setClassificationEnabled($accountData['classificationEnabled'] ?? false);
149-
$newAccount->setSignatureAboveQuote($accountData['signatureAboveQuote'] ?? false);
150-
$newAccount->setPersonalNamespace($accountData['personalNamespace'] ?? null);
151-
if (isset($accountData['inboundPassword'])) {
152-
$newAccount->setInboundPassword($this->crypto->encrypt($accountData['inboundPassword']));
153-
}
154-
if (isset($accountData['outboundPassword'])) {
155-
$newAccount->setOutboundPassword($this->crypto->encrypt($accountData['outboundPassword']));
156-
}
157-
if (isset($accountData['oauthRefreshToken'])) {
158-
$newAccount->setOauthRefreshToken($this->crypto->encrypt($accountData['oauthRefreshToken']));
159-
}
160-
if (isset($accountData['oauthAccessToken'])) {
161-
$newAccount->setOauthAccessToken($this->crypto->encrypt($accountData['oauthAccessToken']));
162-
}
163-
$newAccount->setOauthTokenTtl($accountData['oauthTokenTtl'] ?? null);
164-
165-
$mailAccount = $this->accountService->save(
166-
$newAccount
167-
);
168-
169-
// Import aliases
170-
foreach ($accountData['aliases'] as $alias) {
171-
$this->aliasesService->create(
172-
$user->getUID(),
173-
$mailAccount->getId(),
174-
$alias['alias'],
175-
$alias['name'],
176-
);
177-
}
178-
}
45+
$output->writeln("Importing mail accounts for user {$user->getUID()}", OutputInterface::VERBOSITY_VERBOSE);
46+
47+
$this->deleteExistingData($user, $output);
48+
49+
$this->accountMigrationService->scheduleBackgroundJobs($user, $output);
50+
}
51+
52+
/**
53+
* Delete all existing user data of our app to ensure
54+
* the result of the import is always the same.
55+
*
56+
* @param IUser $user
57+
* @param OutputInterface $output
58+
* @throws ClientException
59+
* @throws DoesNotExistException
60+
* @throws ServiceException
61+
*/
62+
private function deleteExistingData(IUser $user, OutputInterface $output): void {
63+
$output->writeln("Deleting existing mail data for user {$user->getUID()}", OutputInterface::VERBOSITY_VERBOSE);
64+
$this->accountMigrationService->deleteAllAccounts($user, $output);
17965
}
18066

18167
#[\Override]
@@ -195,7 +81,7 @@ public function getDescription(): string {
19581

19682
#[\Override]
19783
public function getVersion(): int {
198-
return 01_00_00;
84+
return 02_00_00;
19985
}
20086

20187
#[\Override]

‎lib/UserMigration/Service/AccountMigrationService.php‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OCA\Mail\UserMigration\Service;
11+
12+
use OCA\Mail\Exception\ClientException;
13+
use OCA\Mail\Service\AccountService;
14+
use OCP\IUser;
15+
use Symfony\Component\Console\Output\OutputInterface;
16+
17+
class AccountMigrationService {
18+
public function __construct(
19+
private readonly AccountService $accountService,
20+
) {
21+
}
22+
23+
/**
24+
* Delete all mail accounts for the given user.
25+
*
26+
* @param IUser $user
27+
* @param OutputInterface $output
28+
* @return void
29+
* @throws ClientException
30+
*/
31+
public function deleteAllAccounts(IUser $user, OutputInterface $output): void {
32+
$allAccounts = $this->accountService->findByUserId($user->getUID());
33+
$accountCount = count($allAccounts);
34+
$uid = $user->getUID();
35+
36+
$output->writeln("Deleting {$accountCount} mail account(s) for user {$uid}", OutputInterface::VERBOSITY_VERBOSE);
37+
38+
foreach ($allAccounts as $account) {
39+
$accountId = $account->getId();
40+
41+
if ($account->getMailAccount()->getProvisioningId() !== null) {
42+
$output->writeln("Skipping deletion of provisioned account {$account->getId()}", OutputInterface::VERBOSITY_VERBOSE);
43+
continue;
44+
}
45+
46+
$this->accountService->deleteByAccountId($accountId);
47+
$output->writeln("Deleted mail account {$accountId}", OutputInterface::VERBOSITY_VERBOSE);
48+
}
49+
}
50+
51+
/**
52+
* Schedule background jobs for the added accounts.
53+
* Necessary to do after all data is being imported as we
54+
* could run into race conditions when doing directly after
55+
* saving each mail account into database.
56+
*
57+
* @param IUser $user
58+
* @param OutputInterface $output
59+
* @return void
60+
*/
61+
public function scheduleBackgroundJobs(IUser $user, OutputInterface $output): void {
62+
$accounts = $this->accountService->findByUserId($user->getUID());
63+
$accountCount = count($accounts);
64+
65+
$output->writeln("Scheduling background jobs for {$accountCount} mail account(s)", OutputInterface::VERBOSITY_VERBOSE);
66+
67+
foreach ($accounts as $account) {
68+
$mailAccount = $account->getMailAccount();
69+
$mailAccountId = $mailAccount->getId();
70+
$this->accountService->scheduleBackgroundJobs($mailAccountId);
71+
$output->writeln("Scheduled background jobs for mail account {$mailAccountId}", OutputInterface::VERBOSITY_VERY_VERBOSE);
72+
}
73+
}
74+
}

0 commit comments

Comments
 (0)