fix: sort zoids in CacheWarmer._flush to prevent PK-index deadlock (#61)

Concurrent Waitress workers with overlapping pending sets deadlocked
on cache_warm_stats PK index when their INSERT ... ON CONFLICT DO
UPDATE acquired row locks in opposing orders. Set iteration order is
not stable across processes, so two flushes could see the same zoids
in different orders.

Sort zoids before the upsert for a deterministic row-lock acquisition
order. Added regression test verifying the INSERT parameter is sorted.

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: jensens

CHANGES.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 1.11.1
+
+- Fix PK-index deadlock in `CacheWarmer._flush` between concurrent
+  Waitress workers.  Two workers with overlapping pending sets used to
+  deadlock on the `cache_warm_stats` primary-key index when their
+  `INSERT ... ON CONFLICT DO UPDATE` acquired row locks in opposing
+  orders (set iteration order is not stable across processes).  Zoids
+  are now sorted before the upsert, giving a deterministic lock
+  acquisition order.
+
 ## 1.11.0
 
 - Implement `IStorage.afterCompletion()` on `PGJsonbStorageInstance`

src/zodb_pgjsonb/cache_warmer.py

@@ -84,8 +84,13 @@ def _flush(self, decay=False):
 
         Wrapped in an explicit transaction so the decay UPDATE, score
         UPSERT, and low-score DELETE are atomic.
+
+        Zoids are sorted to give a deterministic row-lock acquisition
+        order across concurrent flushes. Without this, two workers with
+        overlapping pending sets deadlock on the PK index when they
+        upsert rows in opposing orders.
         """
-        zoids = list(self._pending)
+        zoids = sorted(self._pending)
         if not zoids:
             return
         self._pending.clear()

tests/test_cache_warmer.py

@@ -297,6 +297,34 @@ def test_flush_exception_triggers_rollback(self):
         assert "BEGIN" in calls
         assert "ROLLBACK" in calls
 
+    def test_flush_sorts_zoids_for_deterministic_locking(self):
+        """Prevent PK-index deadlock between concurrent workers.
+
+        The INSERT must receive zoids in sorted order so that two
+        workers with overlapping pending sets acquire row locks in the
+        same order and never deadlock.
+        """
+        from zodb_pgjsonb.cache_warmer import CacheWarmer
+
+        conn = mock.MagicMock()
+        cursor = conn.cursor.return_value.__enter__.return_value
+
+        w = CacheWarmer(conn=conn, target_count=100)
+        w._pending = {42, 7, 99, 3, 58}
+        w._flush(decay=False)
+
+        insert_calls = [
+            call
+            for call in cursor.execute.call_args_list
+            if "INSERT INTO cache_warm_stats" in call.args[0]
+        ]
+        assert len(insert_calls) == 1
+        zoids_param = insert_calls[0].args[1]["z"]
+        assert zoids_param == sorted(zoids_param), (
+            f"zoids must be sorted for deterministic lock order, got {zoids_param}"
+        )
+        assert zoids_param == [3, 7, 42, 58, 99]
+
 
 class TestWarmLoadMultiple:
     """Integration test for PGJsonbStorage._warm_load_multiple()."""