[stable33] Fix removed address book items not being synced between federated instances

apps/dav/lib/CardDAV/CardDavBackend.php

@@ -478,6 +478,13 @@
 			->from($this->dbCardsTable)
 			->where($query->expr()->eq('addressbookid', $query->createNamedParameter($addressbookId)));
 
+		return $this->getCardsFromQuery($query);
+	}
+
+	/**
+	 * @return array[]
+	 */
+	private function getCardsFromQuery(IQueryBuilder $query): array {
 		$cards = [];
 
 		$result = $query->executeQuery();
@@ -972,7 +979,8 @@
 					->from('cards')
 					->where(
 						$qb->expr()->eq('addressbookid', $qb->createNamedParameter($addressBookId))
-					);
+					)
+					->orderBy('id');
 				// No synctoken supplied, this is the initial sync.
 				$qb->setMaxResults($limit);
 				$stmt = $qb->executeQuery();
@@ -1532,4 +1540,32 @@
 		// should already be handled, but just in case
 		throw new BadRequest('vCard can not be empty');
 	}
+
+	/**
+	 * Mark all cards in an address book as needing to be validated
+	 *
+	 * This is done by setting the modified date to `null`, once a sync runs
+	 * the mtime will be set to a non-null value. Leaving all deleted items with
+	 * a null modified date.
+	 */
+	public function markCardsAsPending(int $addressBookId): void {
+		$query = $this->db->getTypedQueryBuilder();
+		$query->update($this->dbCardsTable)
+			->set('lastmodified', $query->createNamedParameter(null))
+			->where($query->expr()->eq('addressbookid', $query->createNamedParameter($addressBookId)))
+			->executeStatement();
+	}
+
+	/**
+	 * @return array[]
+	 */
+	public function getPendingCards(int $addressBookId): array {
+		$query = $this->db->getQueryBuilder();
+		$query->select(['id', 'addressbookid', 'uri', 'lastmodified', 'etag', 'size', 'carddata', 'uid'])
+			->from($this->dbCardsTable)
+			->where($query->expr()->eq('addressbookid', $query->createNamedParameter($addressBookId)))
+			->andWhere($query->expr()->isNull('lastmodified'));
+
+		return $this->getCardsFromQuery($query);
+	}
 }

apps/dav/lib/CardDAV/SyncService.php

@@ -214,4 +214,15 @@ public function syncInstance(?\Closure $progressCallback = null) {
 	public static function getCardUri(IUser $user): string {
 		return $user->getBackendClassName() . ':' . $user->getUID() . '.vcf';
 	}
+
+	public function markCardsAsPending(int $addressBookId): void {
+		$this->backend->markCardsAsPending($addressBookId);
+	}
+
+	public function deletePendingCards(int $addressBookId): void {
+		$cards = $this->backend->getPendingCards($addressBookId);
+		foreach ($cards as $card) {
+			$this->backend->deleteCard($addressBookId, $card['uri']);
+		}
+	}
 }

apps/dav/lib/CardDAV/SystemAddressbook.php

@@ -232,7 +232,8 @@ public function getChanges($syncToken, $syncLevel, $limit = null) {
 			return $changed;
 		}
 
-		$added = $modified = $deleted = [];
+		$added = $modified = [];
+		$deleted = array_values($changed['deleted']);
 		foreach ($changed['added'] as $uri) {
 			try {
 				$this->getChild($uri);

apps/dav/tests/unit/CardDAV/SyncServiceTest.php

@@ -5,6 +5,7 @@
  * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
  * SPDX-License-Identifier: AGPL-3.0-only
  */
+
 namespace OCA\DAV\Tests\unit\CardDAV;
 
 use GuzzleHttp\Exception\ClientException;
@@ -104,7 +105,7 @@ public function testEmptySync(): void {
 			'system',
 			'system',
 			'1234567890',
-			null,
+			'1',
 			'1',
 			'principals/system/system',
 			[]
@@ -175,7 +176,7 @@ public function testSyncWithNewElement(): void {
 			'system',
 			'system',
 			'1234567890',
-			null,
+			'1',
 			'1',
 			'principals/system/system',
 			[]
@@ -246,7 +247,7 @@ public function testSyncWithUpdatedElement(): void {
 			'system',
 			'system',
 			'1234567890',
-			null,
+			'1',
 			'1',
 			'principals/system/system',
 			[]
@@ -287,7 +288,7 @@ public function testSyncWithDeletedElement(): void {
 			'system',
 			'system',
 			'1234567890',
-			null,
+			'1',
 			'1',
 			'principals/system/system',
 			[]
@@ -296,6 +297,97 @@ public function testSyncWithDeletedElement(): void {
 		$this->assertEquals('http://sabre.io/ns/sync/4', $token);
 	}
 
+	public function testFullSyncWithOrphanElement(): void {
+		$pendingCards = [];
+		$this->backend->expects($this->exactly(0))
+			->method('createCard');
+		$this->backend->expects($this->exactly(1))
+			->method('updateCard')
+			->willReturnCallback(function ($id, $uri) use (&$pendingCards) {
+				unset($pendingCards[$uri]);
+			});
+		$this->backend->expects($this->exactly(1))
+			->method('markCardsAsPending')
+			->willReturnCallback(function ($id) use (&$pendingCards) {
+				$cards = array_values($this->backend->getCards($id));
+				$uris = array_map(fn ($card) => $card['uri'], $cards);
+				$pendingCards = array_combine($uris, $cards);
+			});
+		$this->backend->expects($this->exactly(1))
+			->method('getPendingCards')
+			->willReturnCallback(function ($id) use (&$pendingCards) {
+				return array_values($pendingCards);
+			});
+		$this->backend->expects($this->exactly(1))
+			->method('deleteCard');
+
+		$body = '<?xml version="1.0"?>
+<d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns" xmlns:card="urn:ietf:params:xml:ns:carddav" xmlns:oc="http://owncloud.org/ns">
+    <d:response>
+        <d:href>/remote.php/dav/addressbooks/system/system/system/Database:alice.vcf</d:href>
+        <d:propstat>
+            <d:prop>
+                <d:getcontenttype>text/vcard; charset=utf-8</d:getcontenttype>
+                <d:getetag>&quot;2df155fa5c2a24cd7f750353fc63f037&quot;</d:getetag>
+            </d:prop>
+            <d:status>HTTP/1.1 200 OK</d:status>
+        </d:propstat>
+    </d:response>
+    <d:sync-token>http://sabre.io/ns/sync/3</d:sync-token>
+</d:multistatus>';
+
+		$reportResponse = new Response(new PsrResponse(
+			207,
+			['Content-Type' => 'application/xml; charset=utf-8', 'Content-Length' => strlen($body)],
+			$body
+		));
+
+		$this->client
+			->method('request')
+			->willReturn($reportResponse);
+
+		$vCard = 'BEGIN:VCARD
+VERSION:3.0
+PRODID:-//Sabre//Sabre VObject 4.5.4//EN
+UID:alice
+FN;X-NC-SCOPE=v2-federated:alice
+N;X-NC-SCOPE=v2-federated:alice;;;;
+X-SOCIALPROFILE;TYPE=NEXTCLOUD;X-NC-SCOPE=v2-published:https://server2.internal/index.php/u/alice
+CLOUD:alice@server2.internal
+END:VCARD';
+
+		$getResponse = new Response(new PsrResponse(
+			200,
+			['Content-Type' => 'text/vcard; charset=utf-8', 'Content-Length' => strlen($vCard)],
+			$vCard,
+		));
+
+		$this->client
+			->method('get')
+			->willReturn($getResponse);
+
+		$this->backend->method('getCards')
+			->willReturn([
+				['uri' => 'Database:alice.vcf'],
+				['uri' => 'Database:bob.vcf'],
+			]);
+
+		$this->service->markCardsAsPending(1);
+		$token = $this->service->syncRemoteAddressBook(
+			'',
+			'system',
+			'system',
+			'1234567890',
+			null,
+			'1',
+			'principals/system/system',
+			[]
+		)[0];
+		$this->service->deletePendingCards(1);
+
+		$this->assertEquals('http://sabre.io/ns/sync/3', $token);
+	}
+
 	public function testEnsureSystemAddressBookExists(): void {
 		/** @var CardDavBackend&MockObject $backend */
 		$backend = $this->createMock(CardDavBackend::class);
@@ -458,7 +550,7 @@ public function testUseAbsoluteUriReport(string $host, string $expected): void {
 			'system',
 			'remote.php/dav/addressbooks/system/system/system',
 			'1234567890',
-			null,
+			'1',
 			'1',
 			'principals/system/system',
 			[]

apps/federation/lib/Command/SyncFederationAddressBooks.php

@@ -11,6 +11,7 @@
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 class SyncFederationAddressBooks extends Command {
@@ -23,19 +24,21 @@ public function __construct(
 	protected function configure() {
 		$this
 			->setName('federation:sync-addressbooks')
-			->setDescription('Synchronizes addressbooks of all federated clouds');
+			->setDescription('Synchronizes addressbooks of all federated clouds')
+			->addOption('full', null, InputOption::VALUE_NONE, 'Perform a full sync instead of a delta sync');
 	}
 
 	protected function execute(InputInterface $input, OutputInterface $output): int {
 		$progress = new ProgressBar($output);
 		$progress->start();
+		$full = (bool)$input->getOption('full');
 		$this->syncService->syncThemAll(function ($url, $ex) use ($progress, $output): void {
 			if ($ex instanceof \Exception) {
 				$output->writeln("Error while syncing $url : " . $ex->getMessage());
 			} else {
 				$progress->advance();
 			}
-		});
+		}, $full);
 
 		$progress->finish();
 		$output->writeln('');

apps/federation/lib/SyncFederationAddressBooks.php

@@ -28,7 +28,7 @@ public function __construct(
 	/**
 	 * @param \Closure $callback
 	 */
-	public function syncThemAll(\Closure $callback) {
+	public function syncThemAll(\Closure $callback, bool $full = false) {
 		$trustedServers = $this->dbHandler->getAllServer();
 		foreach ($trustedServers as $trustedServer) {
 			$url = $trustedServer['url'];
@@ -51,7 +51,12 @@ public function syncThemAll(\Closure $callback) {
 			];
 
 			try {
-				$syncToken = $oldSyncToken;
+				$syncToken = $full ? null : $oldSyncToken;
+
+				$book = $this->syncService->ensureSystemAddressBookExists($targetPrincipal, $targetBookId, $targetBookProperties);
+				if ($full) {
+					$this->syncService->markCardsAsPending($book['id']);
+				}
 
 				do {
 					[$syncToken, $truncated] = $this->syncService->syncRemoteAddressBook(
@@ -66,6 +71,10 @@ public function syncThemAll(\Closure $callback) {
 					);
 				} while ($truncated);
 
+				if ($full) {
+					$this->syncService->deletePendingCards($book['id']);
+				}
+
 				if ($syncToken !== $oldSyncToken) {
 					$this->dbHandler->setServerStatus($url, TrustedServers::STATUS_OK, $syncToken);
 				} else {