misc changes done while working on a server checker script

Author: tobixen

CHANGELOG.md

@@ -10,20 +10,18 @@ This project should adhere to [Semantic Versioning](https://semver.org/spec/v2.0
 
 In version 2.0, the requests library will be replaced with niquests or httpx.  See https://github.com/python-caldav/caldav/issues/457.  Master branch is currently running niquests.
 
-(Also, in version 2.0, support for python 3.7 will be dropped, possibly also python 3.8.  Master branch supports both as for now).
+In version 2.0, support for python 3.7 will be dropped, possibly also python 3.8.  Master branch supports both as for now.
 
 ## [1.5.0] - [Unreleased]
 
-### Fixed
-
-* Readthedocs integration has been repaired (https://github.com/python-caldav/caldav/pull/453 - but eventually the fix was introduced directly in the master branch)
-
 ### Deprecated
 
 Python 3.7 is no longer tested - but it should work.  Please file a bug report if it doesn't work.  (Note that the caldav library pulls in many dependencies, and not all of them supports dead snakes).
 
 ### Changed
 
+* Some exotic servers may return object URLs on search, but it does not work out to fetch the calendar data.  Now it will log an error instead of raising an error in such cases.
+
 #### Refactoring
 
 * Minor code cleanups by github user @ArtemIsmagilov in https://github.com/python-caldav/caldav/pull/456
@@ -38,15 +36,19 @@ Python 3.7 is no longer tested - but it should work.  Please file a bug report i
     * Creating a local xandikos or radicale server in the `tests.client`-method, which is also used in the `examples`-section.
     * Allows offline testing of my upcoming `check_server_compatibility`-script
   * Also added the possibility to tag test servers with a name
+* Many changes done to the compatibility flag list (due to work on the server-checker project)
 
 ### Added
 
-* By now `calendar.search(..., sort_keys=("DTSTART")` will work.  Sort keys expects a list or a tuple, but it's easy to send an attribute by mistake.  https://github.com/python-caldav/caldav/pull/449
+* By now `calendar.search(..., sort_keys=("DTSTART")` will work.  Sort keys expects a list or a tuple, but it's easy to send an attribute by mistake.  https://github.com/python-caldav/caldav/issues/448 https://github.com/python-caldav/caldav/pull/449
 * Compatibility workaround: If `event.load()` fails, it will retry the load by doing a multiget - https://github.com/python-caldav/caldav/pull/475 - https://github.com/python-caldav/caldav/issues/459
+* The `class_`-parameter now works when sending data to `save_event()` etc.
 
 ### Fixed
 
 * Bugfix for saving component failing on multi-component recurrence objects - https://github.com/python-caldav/caldav/pull/467
+* Readthedocs integration has been repaired (https://github.com/python-caldav/caldav/pull/453 - but eventually the fix was introduced directly in the master branch)
+
 
 ## [1.4.0] - 2024-11-05
 

caldav/davclient.py

@@ -207,6 +207,7 @@ def _parse_response(self, response) -> Tuple[str, List[_Element], Optional[Any]]
         status = None
         href: Optional[str] = None
         propstats: List[_Element] = []
+        check_404 = False  ## special for purelymail
         error.assert_(response.tag == dav.Response.tag)
         for elem in response:
             if elem.tag == dav.Status.tag:
@@ -223,13 +224,29 @@ def _parse_response(self, response) -> Tuple[str, List[_Element], Optional[Any]]
                 href = unquote(elem.text)
             elif elem.tag == dav.PropStat.tag:
                 propstats.append(elem)
+            elif elem.tag == "{DAV:}error":
+                ## This happens with purelymail on a 404.
+                ## This code is mostly moot, but in debug
+                ## mode I want to be sure we do not toss away any data
+                children = elem.getchildren()
+                error.assert_(len(children) == 1)
+                error.assert_(
+                    children[0].tag == "{https://purelymail.com}does-not-exist"
+                )
+                check_404 = True
             else:
-                error.assert_(False)
+                ## i.e. purelymail may contain one more tag, <error>...</error>
+                ## This is probably not a breach of the standard.  It may
+                ## probably be ignored.  But it's something we may want to
+                ## know.
+                error.weirdness("unexpected element found in response", elem)
         error.assert_(href)
+        if check_404:
+            error.assert_("404" in status)
         ## TODO: is this safe/sane?
         ## Ref https://github.com/python-caldav/caldav/issues/435 the paths returned may be absolute URLs,
         ## but the caller expects them to be paths.  Could we have issues when a server has same path
-        ## but different URLs for different elements?
+        ## but different URLs for different elements?  Perhaps href should always be made into an URL-object?
         if ":" in href:
             href = unquote(URL(href).path)
         return (cast(str, href), propstats, status)

caldav/lib/debug.py

@@ -2,6 +2,8 @@
 
 
 def xmlstring(root):
+    if isinstance(root, str):
+        return root
     if hasattr(root, "xmlelement"):
         root = root.xmlelement()
     return etree.tostring(root, pretty_print=True).decode("utf-8")

caldav/lib/error.py

@@ -14,7 +14,7 @@
     ## one of DEBUG_PDB, DEBUG, DEVELOPMENT, PRODUCTION
     debugmode = os.environ["PYTHON_CALDAV_DEBUGMODE"]
 except:
-    if "dev" in __version__:
+    if "dev" in __version__ or __version__ == "(unknown)":
         debugmode = "DEVELOPMENT"
     else:
         debugmode = "PRODUCTION"
@@ -26,6 +26,18 @@
     log.setLevel(logging.WARNING)
 
 
+def weirdness(*reasons):
+    from caldav.lib.debug import xmlstring
+
+    reason = " : ".join([xmlstring(x) for x in reasons])
+    log.warning(f"Deviation from expectations found: {reason}")
+    if debugmode == "DEBUG_PDB":
+        log.error(f"Dropping into debugger due to {reason}")
+        import pdb
+
+        pdb.set_trace()
+
+
 def assert_(condition: object) -> None:
     try:
         assert condition

caldav/lib/vcal.py

@@ -154,6 +154,8 @@ def create_ical(ical_fragment=None, objtype=None, language="en_DK", **props):
     I somehow feel this fits more into the icalendar library than here
     """
     ical_fragment = to_normal_str(ical_fragment)
+    if "class_" in props:
+        props["class"] = props.pop("class_")
     if not ical_fragment or not re.search("^BEGIN:V", ical_fragment, re.MULTILINE):
         my_instance = icalendar.Calendar()
         if objtype is None:

caldav/objects.py

@@ -794,7 +794,7 @@ def _create(
         if name:
             try:
                 self.set_properties([display_name])
-            except:
+            except Exception as e:
                 ## TODO: investigate.  Those asserts break.
                 error.assert_(False)
                 try:
@@ -1232,9 +1232,19 @@ def search(
                     )
                 raise
 
+        obj2 = []
         for o in objects:
             ## This would not be needed if the servers would follow the standard ...
-            o.load(only_if_unloaded=True)
+            try:
+                o.load(only_if_unloaded=True)
+                obj2.append(o)
+            except:
+                logging.error(
+                    "Server does not want to reveal details about the calendar object",
+                    exc_info=True,
+                )
+                pass
+        objects = obj2
 
         ## Google sometimes returns empty objects
         objects = [o for o in objects if o.has_component()]

tests/compatibility_issues.py

@@ -15,6 +15,9 @@
     'rate_limited':
         """It may be needed to pause a bit between each request when doing tests""",
 
+    'search_delay':
+        """Server populates indexes through some background job, so it takes some time from an event is added/edited until it's possible to search for it""",
+
     'cleanup_calendar':
         """Remove everything on the calendar for every test""",
 
@@ -30,6 +33,9 @@
     'broken_expand_on_exceptions':
         """The testRecurringDateWithExceptionSearch test breaks as the icalendar_component is missing a RECURRENCE-ID field.  TODO: should be investigated more""",
 
+    'inaccurate_datesearch':
+        """A date search may yield results outside the search interval""",
+
     'no_current-user-principal':
         """Current user principal not supported by the server (flag is ignored by the tests as for now - pass the principal URL as the testing URL and it will work, albeit with one warning""",
 
@@ -92,6 +98,9 @@
     'event_by_url_is_broken':
         """A GET towards a valid calendar object resource URL will yield 404 (wtf?)""",
 
+    'no_delete_event':
+        """Zimbra does not support deleting an event, probably because event_by_url is broken""",
+
     'no_sync_token':
         """RFC6578 is not supported, things will break if we try to do a sync-token report""",
 
@@ -121,6 +130,9 @@
     'no_todo':
         """Support for VTODO (tasks) apparently missing""",
 
+    'no_todo_on_standard_calendar':
+        """Tasklists can be created, but a normal calendar does not support tasks""",
+
     'no_todo_datesearch':
         """Date search on todo items fails""",
 
@@ -134,15 +146,27 @@
         """date searches for todo-items will (only) find tasks that has either """
         """a dtstart or due set""",
 
+    'vtodo_datesearch_nostart_future_tasks_delivered':
+        """Future tasks are yielded when doing a date search with some end timestamp and without start timestamp and the task contains both dtstart and due, but not duration (xandikos 0.2.12)""",
+
     'vtodo_no_due_infinite_duration':
         """date search will find todo-items without due if dtstart is """
-        """before the date search interval.  I didn't find anything explicit """
-        """in The RFC on this (), but an event should be considered to have 0 """
-        """duration if no dtend is set, and most server implementations seems to """
-        """treat VTODOs the same""",
+        """before the date search interval.  This is in breach of rfc4791"""
+        """section 9.9""",
 
-    'no_todo_on_standard_calendar':
-        """Tasklists can be created, but a normal calendar does not support tasks""",
+    'vtodo_no_dtstart_infinite_duration':
+        """date search will find todo-items without dtstart if due is """
+        """after the date search interval.  This is in breach of rfc4791"""
+        """section 9.9""",
+
+    'vtodo_no_dtstart_search_weirdness':
+       """Zimbra is weird""",
+
+    'vtodo_no_duration_search_weirdness':
+       """Zimbra is weird""",
+
+    'vtodo_with_due_weirdness':
+       """Zimbra is weird""",
 
     'unique_calendar_ids':
         """For every test, generate a new and unique calendar id""",
@@ -181,8 +205,11 @@
    'text_search_not_working':
         """Text search is generally broken""",
 
-   'radicale_breaks_on_category_search':
-        """See https://github.com/Kozea/Radicale/issues/1125""",
+    'date_search_ignores_duration':
+        """Date search with search interval overlapping event interval works on events with dtstart and dtend, but not on events with dtstart and due""",
+
+    'date_todo_search_ignores_duration':
+        """Same as above, but specifically for tasks""",
 
    'fastmail_buggy_noexpand_date_search':
         """The 'blissful anniversary' recurrent example event is returned when asked for a no-expand date search for some timestamps covering a completely different date""",
@@ -226,35 +253,54 @@
     ## https://github.com/jelmer/xandikos/issues/8
     "no_recurring",
 
+    'date_todo_search_ignores_duration',
     'text_search_is_exact_match_only',
-
-    ## This one is fixed - but still breaks our test code for python 3.7
-    ## TODO: remove this when shredding support for python 3.7
-    ## https://github.com/jelmer/xandikos/pull/194
-    'category_search_yields_nothing',
+    "search_needs_comptype",
+    'vtodo_datesearch_nostart_future_tasks_delivered',
 
     ## scheduling is not supported
     "no_scheduling",
+
+    ## The test in the tests itself passes, but the test in the
+    ## check_server_compatibility triggers a 500-error
+    "no_freebusy_rfc4791",
+
+    ## The test with an rrule and an overridden event passes as
+    ## long as it's with timestamps.  With dates, xandikos gets
+    ## into troubles.  I've chosen to edit the test to use timestamp
+    ## rather than date, just to have the test exercised ... but we
+    ## should report this upstream
+    #'broken_expand_on_exceptions',
 ]
 
+## This can soon be removed (relevant for running tests under python 3.7 and python 3.8)
+## https://github.com/jelmer/xandikos/pull/194
+'category_search_yields_nothing',
+try:
+    from xandikos import __version__ as xver
+    goodver = (0,2,12)
+    for i in range(0,3):
+        if xver[i]<goodver[i]:
+            xandikos.append('category_search_yields_nothing')
+            break
+except Exception:
+    pass
+
 radicale = [
     ## calendar listings and calendar creation works a bit
     ## "weird" on radicale
     "broken_expand",
     "no_default_calendar",
 
     ## freebusy is not supported yet, but on the long-term road map
-    "no_freebusy_rfc4791",
-
-    ## TODO: raise an issue on this one
-    "radicale_breaks_on_category_search",
+    #"no_freebusy_rfc4791",
 
     'no_scheduling',
-    'no_todo_datesearch',
+    "no_todo_datesearch",
 
     'text_search_is_case_insensitive',
-    'text_search_is_exact_match_sometimes',
-    'combined_search_not_working',
+    #'text_search_is_exact_match_sometimes',
+    "search_needs_comptype",
 
     ## extra features not specified in RFC5545
     "calendar_order",
@@ -276,10 +322,9 @@
     ## earlier versions of Zimbra display-name could be changed, but
     ## then the calendar would not be available on the old URL
     ## anymore)
-    'no_displayname',
     'duplicate_in_other_calendar_with_same_uid_is_lost',
     'event_by_url_is_broken',
-    'no_todo_on_standard_calendar',
+    'no_delete_event',
     'no_sync_token',
     'vtodo_datesearch_notime_task_is_skipped',
     'category_search_yields_nothing',
@@ -341,7 +386,11 @@
     #'nofreebusy', ## for old versions
     'fragile_sync_tokens', ## no issue raised yet
     'vtodo_datesearch_nodtstart_task_is_skipped', ## no issue raised yet
-    'broken_expand_on_exceptions',
+    'broken_expand_on_exceptions', ## no issue raised yet
+    'date_todo_search_ignores_duration'
+    'calendar_color',
+    'calendar_order',
+    'vtodo_datesearch_notime_task_is_skipped'
 ]
 
 google = [
@@ -366,10 +415,16 @@
 ]
 
 nextcloud = [
+    'date_search_ignores_duration',
     'sync_breaks_on_delete',
     'no_recurring_todo',
     'combined_search_not_working',
     'text_search_is_exact_match_sometimes',
+    'search_needs_comptype',
+    'calendar_color',
+    'calendar_order',
+    'date_todo_search_ignores_duration',
+    'broken_expand_on_exceptions'
 ]
 
 fastmail = [
@@ -440,11 +495,7 @@
 
     ## Purelymail claims that the search indexes are "lazily" populated,
     ## so search works some minutes after the event was created/edited.
-    ## I tried adding arbitrary delays in commit 5d052b1 but still didn't
-    ## manage to get search to work.  Should eventually do more research
-    ## into this.  (personal email communication with contact@purelymail.com)
-    'no_search',
-    'object_by_uid_is_broken',
+    'search_delay'
 ]
 
 # fmt: on