feat: bidirectional CardDAV contact sync

.github/workflows/ci.yml

@@ -11,6 +11,24 @@ jobs:
     name: Backend
     runs-on: ubuntu-latest
 
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres -d postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+
+    env:
+      TEST_DATABASE_URL: postgres://postgres:postgres@127.0.0.1:5432/postgres
+
     steps:
       - uses: actions/checkout@v6
 

backend/migrations/0034_carddav_incremental_sync.sql

@@ -0,0 +1,76 @@
+-- Remote CardDAV collection state is separate from the sync token and ETags
+-- Mailflow serves to its own CardDAV clients.
+DO $$
+BEGIN
+  IF EXISTS (
+    WITH ranked AS (
+      SELECT id,
+             row_number() OVER (
+               PARTITION BY user_id, external_url
+               ORDER BY created_at, id
+             ) AS position
+      FROM address_books
+      WHERE source = 'carddav' AND external_url IS NOT NULL
+    )
+    SELECT 1
+    FROM ranked
+    JOIN contacts contact ON contact.address_book_id = ranked.id
+    WHERE ranked.position > 1
+  ) THEN
+    RAISE EXCEPTION 'cannot consolidate populated duplicate CardDAV address books';
+  END IF;
+END $$;
+
+ALTER TABLE address_books
+  ADD COLUMN IF NOT EXISTS remote_sync_token TEXT,
+  ADD COLUMN IF NOT EXISTS remote_sync_capability TEXT NOT NULL DEFAULT 'unknown'
+    CHECK (remote_sync_capability IN ('unknown', 'sync-collection', 'snapshot')),
+  ADD COLUMN IF NOT EXISTS remote_sync_revision BIGINT NOT NULL DEFAULT 0,
+  ADD COLUMN IF NOT EXISTS remote_projection_fingerprint TEXT;
+
+CREATE TABLE IF NOT EXISTS carddav_remote_objects (
+  address_book_id UUID NOT NULL REFERENCES address_books(id) ON DELETE CASCADE,
+  href TEXT NOT NULL,
+  remote_etag TEXT,
+  vcard TEXT NOT NULL,
+  primary_email TEXT,
+  disposition TEXT NOT NULL CHECK (disposition IN ('separate', 'merge', 'skip')),
+  local_contact_id UUID REFERENCES contacts(id) ON DELETE SET NULL,
+  merge_before JSONB,
+  merge_applied JSONB,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  PRIMARY KEY (address_book_id, href)
+);
+
+UPDATE user_integrations
+SET config = config || jsonb_build_object('connectionGeneration', gen_random_uuid()::text)
+WHERE provider = 'carddav'
+  AND NOT (config ? 'connectionGeneration');
+
+WITH ranked AS (
+  SELECT id,
+         row_number() OVER (
+           PARTITION BY user_id, external_url
+           ORDER BY created_at, id
+         ) AS position
+  FROM address_books
+  WHERE source = 'carddav' AND external_url IS NOT NULL
+)
+DELETE FROM address_books
+WHERE id IN (SELECT id FROM ranked WHERE position > 1);
+
+CREATE UNIQUE INDEX IF NOT EXISTS carddav_one_remote_book_idx
+  ON address_books (user_id, external_url)
+  WHERE source = 'carddav' AND external_url IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS carddav_remote_object_contact_idx
+  ON carddav_remote_objects (local_contact_id)
+  WHERE local_contact_id IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS carddav_remote_object_email_idx
+  ON carddav_remote_objects (address_book_id, primary_email);
+
+CREATE UNIQUE INDEX IF NOT EXISTS carddav_one_merge_source_per_contact_idx
+  ON carddav_remote_objects (local_contact_id)
+  WHERE disposition = 'merge' AND local_contact_id IS NOT NULL;

backend/migrations/0035_carddav_bidirectional_sync.sql

@@ -0,0 +1,76 @@
+ALTER TABLE address_books
+  ADD COLUMN IF NOT EXISTS remote_create_capability TEXT NOT NULL DEFAULT 'unknown'
+    CHECK (remote_create_capability IN ('unknown', 'allowed', 'denied')),
+  ADD COLUMN IF NOT EXISTS remote_update_capability TEXT NOT NULL DEFAULT 'unknown'
+    CHECK (remote_update_capability IN ('unknown', 'allowed', 'denied')),
+  ADD COLUMN IF NOT EXISTS remote_delete_capability TEXT NOT NULL DEFAULT 'unknown'
+    CHECK (remote_delete_capability IN ('unknown', 'allowed', 'denied'));
+
+ALTER TABLE contacts
+  ADD COLUMN IF NOT EXISTS additional_fields JSONB NOT NULL DEFAULT '[]'::jsonb;
+
+ALTER TABLE carddav_remote_objects
+  ALTER COLUMN disposition SET DEFAULT 'separate',
+  ADD COLUMN IF NOT EXISTS mapping_status TEXT NOT NULL DEFAULT 'pending_materialization'
+    CHECK (mapping_status IN ('pending_materialization', 'synced', 'pending_push', 'conflict')),
+  ADD COLUMN IF NOT EXISTS vcard_version TEXT
+    CHECK (vcard_version IS NULL OR vcard_version IN ('3.0', '4.0')),
+  ADD COLUMN IF NOT EXISTS remote_semantic_hash TEXT,
+  ADD COLUMN IF NOT EXISTS local_contact_hash TEXT,
+  ADD COLUMN IF NOT EXISTS mapping_revision BIGINT NOT NULL DEFAULT 0
+    CHECK (mapping_revision >= 0),
+  ADD COLUMN IF NOT EXISTS last_synced_at TIMESTAMPTZ,
+  ADD COLUMN IF NOT EXISTS last_push_error_code TEXT,
+  ADD COLUMN IF NOT EXISTS last_push_error_at TIMESTAMPTZ,
+  ADD COLUMN IF NOT EXISTS legacy_projection JSONB;
+
+UPDATE carddav_remote_objects
+SET legacy_projection = jsonb_build_object(
+  'disposition', disposition,
+  'merge_before', merge_before,
+  'merge_applied', merge_applied
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS carddav_one_active_mapping_per_contact_idx
+  ON carddav_remote_objects (local_contact_id)
+  WHERE local_contact_id IS NOT NULL
+    AND mapping_status <> 'pending_materialization';
+
+CREATE TABLE IF NOT EXISTS carddav_conflicts (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  address_book_id UUID NOT NULL,
+  href TEXT NOT NULL,
+  user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+  base_local_hash TEXT,
+  remote_etag TEXT,
+  local_vcard TEXT,
+  remote_vcard TEXT,
+  local_tombstone BOOLEAN NOT NULL DEFAULT false,
+  remote_tombstone BOOLEAN NOT NULL DEFAULT false,
+  status TEXT NOT NULL DEFAULT 'unresolved'
+    CHECK (status IN ('unresolved', 'resolved')),
+  resolution TEXT
+    CHECK (resolution IS NULL OR resolution IN ('keep-mailflow', 'keep-carddav')),
+  resolved_by UUID REFERENCES users(id) ON DELETE SET NULL,
+  resolved_at TIMESTAMPTZ,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  CONSTRAINT carddav_conflicts_remote_object_fkey FOREIGN KEY (address_book_id, href)
+    REFERENCES carddav_remote_objects(address_book_id, href) ON DELETE CASCADE,
+  CHECK (local_tombstone OR local_vcard IS NOT NULL),
+  CHECK (remote_tombstone OR remote_vcard IS NOT NULL),
+  CHECK (
+    (status = 'unresolved'
+      AND resolution IS NULL
+      AND resolved_by IS NULL
+      AND resolved_at IS NULL)
+    OR
+    (status = 'resolved'
+      AND resolution IS NOT NULL
+      AND resolved_at IS NOT NULL)
+  )
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS carddav_one_unresolved_conflict_per_mapping_idx
+  ON carddav_conflicts (address_book_id, href)
+  WHERE status = 'unresolved';

backend/migrations/0036_carddav_bidirectional_cleanup.sql

@@ -0,0 +1,42 @@
+UPDATE user_integrations
+SET config = config - 'dupMode'
+WHERE provider = 'carddav' AND config ? 'dupMode';
+
+DROP INDEX IF EXISTS carddav_one_merge_source_per_contact_idx;
+
+ALTER TABLE carddav_remote_objects
+  DROP COLUMN disposition,
+  DROP COLUMN merge_before,
+  DROP COLUMN merge_applied,
+  DROP COLUMN legacy_projection,
+  ADD COLUMN IF NOT EXISTS pending_operation TEXT
+    CHECK (pending_operation IS NULL OR pending_operation IN ('update', 'delete')),
+  ADD COLUMN IF NOT EXISTS pending_vcard TEXT,
+  ADD COLUMN IF NOT EXISTS pending_local_hash TEXT,
+  ADD COLUMN IF NOT EXISTS pending_remote_semantic_hash TEXT,
+  ADD COLUMN IF NOT EXISTS pending_started_at TIMESTAMPTZ,
+  ADD CHECK (
+    (
+      pending_operation IS NULL
+      AND pending_vcard IS NULL
+      AND pending_local_hash IS NULL
+      AND pending_remote_semantic_hash IS NULL
+      AND pending_started_at IS NULL
+    )
+    OR
+    (
+      pending_operation = 'update'
+      AND pending_vcard IS NOT NULL
+      AND pending_local_hash IS NOT NULL
+      AND pending_remote_semantic_hash IS NOT NULL
+      AND pending_started_at IS NOT NULL
+    )
+    OR
+    (
+      pending_operation = 'delete'
+      AND pending_vcard IS NULL
+      AND pending_local_hash IS NOT NULL
+      AND pending_remote_semantic_hash IS NULL
+      AND pending_started_at IS NOT NULL
+    )
+  );

backend/migrations/0037_carddav_conflict_retention.sql

@@ -0,0 +1,6 @@
+ALTER TABLE carddav_conflicts
+  DROP CONSTRAINT carddav_conflicts_remote_object_fkey;
+
+ALTER TABLE carddav_conflicts
+  ADD CONSTRAINT carddav_conflicts_address_book_id_fkey
+  FOREIGN KEY (address_book_id) REFERENCES address_books(id) ON DELETE CASCADE;

backend/package.json

@@ -5,7 +5,9 @@
   "scripts": {
     "start": "node src/index.js",
     "dev": "node --watch src/index.js",
-    "test": "vitest run",
+    "test": "npm run test:unit && npm run test:db",
+    "test:unit": "vitest run --exclude='**/*.db.test.js' --exclude='**/*.integration.test.js'",
+    "test:db": "vitest run src/**/*.db.test.js src/**/*.integration.test.js",
     "test:watch": "vitest",
     "lint": "eslint src --max-warnings 0"
   },

backend/src/index.js

@@ -29,6 +29,7 @@ import categoriesRoutes from './routes/categories.js';
 import gtdRoutes from './routes/gtd.js';
 import carddavRouter from './routes/carddav.js';
 import carddavAccountRouter from './routes/carddavAccount.js';
+import carddavConflictsRouter from './routes/carddavConflicts.js';
 import { startCardavScheduler } from './services/carddavSync.js';
 import { encryptExistingCredentials, query } from './services/db.js';
 import { runMigrations } from './services/migrations.js';
@@ -159,6 +160,7 @@ app.use('/api/rules', rulesRoutes);
 app.use('/api/block-list', blockListRoutes);
 app.use('/api/contacts', contactsRoutes);
 app.use('/api/todoist', todoistRoutes);
+app.use('/api/carddav/conflicts', carddavConflictsRouter);
 app.use('/api/carddav', carddavAccountRouter);
 app.use('/api', aiRoutes);
 app.use('/api', categoriesRoutes);