compatibility fixing

for servers not supporting combined search (plus radicale), we should ask for all tasks and do client-side filtering.  Also found a weird bug in the test code

Author: tobixen

caldav/collection.py

@@ -887,32 +887,74 @@ def search(
 
         ## special compatibility-case when searching for pending todos
         if todo and not include_completed:
-            matches1 = self.search(
-                todo=True,
-                ignore_completed1=True,
-                include_completed=True,
-                **kwargs,
-            )
-            matches2 = self.search(
-                todo=True,
-                ignore_completed2=True,
-                include_completed=True,
-                **kwargs,
-            )
-            matches3 = self.search(
-                todo=True,
-                ignore_completed3=True,
-                include_completed=True,
-                **kwargs,
-            )
+            ## There are two ways to get the pending tasks - we can
+            ## ask the server to filter them out, or we can do it
+            ## client side.
+
+            ## If the server does not support combined searches, then it's
+            ## safest to do it client-side.
+
+            ## There is a special case (observed with radicale as of
+            ## 2025-11) where future recurrences of a task does not
+            ## match when doing a server-side filtering, so for this
+            ## case we also do client-side filtering (but the
+            ## "feature"
+            ## search.recurrences.includes-implicit.todo.pending will
+            ## not be supported if the feature
+            ## "search.recurrences.includes-implicit.todo" is not
+            ## supported ... hence the weird or below)
+
+            ## To be completely sure to get all pending tasks, for all
+            ## server implementations and for all valid icalendar
+            ## objects, we send three different searches to the
+            ## server.  This is probably bloated, and may in many
+            ## cases be more expensive than to ask for all tasks.  At
+            ## the other hand, for a well-used and well-handled old
+            ## todo-list, there may be a small set of pending tasks
+            ## and heaps of done tasks.
+
+            ## TODO: consider if not ignore_completed3 is sufficient,
+            ## then the recursive part of the query here is moot, and
+            ## we wouldn't waste so much time on repeated queries
+
+            if self.client.features.is_supported("search.combined-is-logical-and") and (
+                not self.client.features.is_supported(
+                    "search.recurrences.includes-implicit.todo"
+                )
+                or self.client.features.is_supported(
+                    "search.recurrences.includes-implicit.todo.pending"
+                )
+            ):
+                matches = (
+                    self.search(
+                        todo=True,
+                        ignore_completed1=True,
+                        include_completed=True,
+                        **kwargs,
+                    )
+                    + self.search(
+                        todo=True,
+                        ignore_completed2=True,
+                        include_completed=True,
+                        **kwargs,
+                    )
+                    + self.search(
+                        todo=True,
+                        ignore_completed3=True,
+                        include_completed=True,
+                        **kwargs,
+                    )
+                )
+            else:
+                matches = self.search(todo=True, include_completed=True, **kwargs)
             objects = []
             match_set = set()
-            for item in matches1 + matches2 + matches3:
+            for item in matches:
                 if item.url not in match_set:
                     match_set.add(item.url)
-                    ## and still, Zimbra seems to deliver too many TODOs in the
-                    ## matches2 ... let's do some post-filtering in case the
-                    ## server fails in filtering things the right way
+                    ## Client-side filtering is probably cheap, so we'll do it
+                    ## even when it shouldn't be needed.
+                    ## (can we assert all tasks have a valid STATUS field?)
                     if any(
                         x.get("STATUS") not in ("COMPLETED", "CANCELLED")
                         for x in item.icalendar_instance.subcomponents

caldav/compatibility_hints.py

@@ -112,12 +112,19 @@ class FeatureSet:
         "search.recurrences.includes-implicit.todo": {
             "description": "tasks can also be recurring"
         },
+        "search.recurrences.includes-implicit.todo.pending": {
+            "description": "a future recurrence of a pending task should always be pending and appear in searches for pending tasks"
+        },
         "search.recurrences.includes-implicit.event": {
             "description": "support for events"
         },
         "search.recurrences.includes-implicit.infinite-scope": {
             "description": "Needless to say, search on any future date range, no matter how far out in the future, should yield the recurring object"
         },
+        "search.combined-is-logical-and": {
+            "description": "Multiple search filters should yield only those that passes all filters"
+            ## For "unsupported", we could also add a "behaviour" (returns everything, returns nothing, returns logical OR, etc).
+        },
         "search.recurrences.expanded": {
             "description": "According to RFC 4791, the server MUST expand recurrence objects if asked for it - but many server doesn't do that.  Some servers don't do expand at all, others deliver broken data, typically missing RECURRENCE-ID.  The python caldav client library (from 2.0) does the expand-operation client-side no matter if it's supported or not",
             "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-9.6.5"],
@@ -540,9 +547,6 @@ def dotted_feature_set_list(self, compact=False):
     'text_search_is_exact_match_sometimes':
         """Some servers are doing an exact match on summary field but substring match on category or vice versa""",
 
-   'combined_search_not_working':
-        """When querying for a text match and a date range in the same report, weird things happen""",
-
    'text_search_not_working':
         """Text search is generally broken""",
 
@@ -626,8 +630,8 @@ def dotted_feature_set_list(self, compact=False):
 ## so I'm expecting this list to shrink a lot soon.
 radicale = {
     "search.category.fullstring": {"support": "unsupported"},
+    "search.recurrences.includes-implicit.todo.pending": {"support": "unsupported"},
     "search.recurrences.expanded.todo": {"support": "unsupported"},
-    "search.recurrences.includes-implicit.todo": {"support": "unsupported"},
     "search.recurrences.expanded.exception": {"support": "unsupported"},
     'old_flags': [
     ## calendar listings and calendar creation works a bit
@@ -640,9 +644,9 @@ def dotted_feature_set_list(self, compact=False):
     "no-principal-search-self", ## this may be because we haven't set up any users or authentication - so the display name of the current user principal is None
 
     'no_scheduling',
+    'no_search_openended',
 
     'text_search_is_case_insensitive',
-    'combined_search_not_working',
     #'text_search_is_exact_match_sometimes',
 
     ## extra features not specified in RFC5545
@@ -668,14 +672,15 @@ def dotted_feature_set_list(self, compact=False):
     'search.comp-type-optional': {
         'support': 'ungraceful',
     },
+    "search.combined-is-logical-and": {"support": "unsupported"},
     'search.recurrences.includes-implicit.todo': {'support': 'unsupported'},
     ## TODO: this applies only to test runs, not to ordinary usage
     'rate-limit': {
         'enable': True,
         'interval': 10,
         'count': 1,
         'description': "It's needed to manually empty trashbin frequently when running tests.  Since this oepration takes some time and/or there are some caches, it's needed to run tests slowly, even when hammering the 'empty thrashbin' frequently"},
-    'old_flags': ['no-principal-search-all', 'no-principal-search-self', 'unique_calendar_ids', 'combined_search_not_working'],
+    'old_flags': ['no-principal-search-all', 'no-principal-search-self', 'unique_calendar_ids'],
 }
 
 ## ZIMBRA IS THE MOST SILLY, AND THERE ARE REGRESSIONS FOR EVERY RELEASE!
@@ -741,11 +746,11 @@ def dotted_feature_set_list(self, compact=False):
     'search.recurrences.expanded.todo': {'support': 'unsupported'},
     'search.recurrences.expanded.exception': {'support': 'unsupported'},
     'search.recurrences.includes-implicit.todo': {'support': 'unsupported'},
+    "search.combined-is-logical-and": {"support": "unsupported"},
     'old_flags': [
         ## date search on todos does not seem to work
         ## (TODO: do some research on this)
         'sync_breaks_on_delete',
-        'combined_search_not_working',
         'text_search_is_exact_match_sometimes',
         ## extra features not specified in RFC5545
         "calendar_order",
@@ -871,12 +876,12 @@ def dotted_feature_set_list(self, compact=False):
     'search.recurrences.expanded.todo': {'support': 'unsupported'},
     'search.recurrences.expanded.exception': {'support': 'unsupported'},
     'search.recurrences.includes-implicit.todo': {'support': 'unsupported'},
+    "search.combined-is-logical-and": {"support": "unsupported"},
     'old_flags': [
         'no_scheduling',
         'no_journal',
         #'no_recurring_todo', ## todo
         'no_sync_token',
-        'combined_search_not_working',
         'no_alarmsearch',
         "no-principal-search-self"
     ]

tests/test_caldav.py

@@ -1767,8 +1767,8 @@ def testSearchEvent(self):
             start=datetime(2006, 7, 13, 13, 0),
             end=datetime(2006, 7, 15, 13, 0),
         )
-        if (self.is_supported("search.category")) and not self.check_compatibility_flag(
-            "combined_search_not_working"
+        if self.is_supported("search.category") and self.is_supported(
+            "search.combined-is-logical-and"
         ):
             assert len(no_events) == 0
         some_events = c.search(
@@ -1777,8 +1777,8 @@ def testSearchEvent(self):
             start=datetime(1997, 11, 1, 13, 0),
             end=datetime(1997, 11, 3, 13, 0),
         )
-        if self.is_supported("search.category") and not self.check_compatibility_flag(
-            "combined_search_not_working"
+        if self.is_supported("search.category") and self.is_supported(
+            "search.combined-is-logical-and"
         ):
             assert len(some_events) == 1
 
@@ -2590,7 +2590,7 @@ def testTodoDatesearch(self):
 
         assert isinstance(todos1[0], Todo)
         assert isinstance(todos2[0], Todo)
-        if not self.check_compatibility_flag("combined_search_not_working"):
+        if not self.check_compatibility_flag("no_search_openended"):
             assert isinstance(todos3[0], Todo)
 
         ## * t6 should be returned, as it's a yearly task spanning over 2025