Explicitly set timespec for isoformat everywhere

Author: Mike

src/hdx/data/dataset.py

@@ -613,7 +613,7 @@ def _save_dataset_add_filestore_resources(
         # No need to output timezone info here so no need to use now_utc()
         self.data[
             "updated_by_script"
-        ] = f"{scriptinfo} ({datetime.utcnow().isoformat()})"
+        ] = f"{scriptinfo} ({datetime.utcnow().isoformat(timespec='microseconds')})"
         batch = kwargs.get("batch")
         if batch:
             if not is_valid_uuid(batch):

src/hdx/data/date_helper.py

@@ -10,13 +10,17 @@
 class DateHelper:
     @staticmethod
     def get_hdx_date(
-        date: Union[datetime, str], ignore_timeinfo: bool, max=False
+        date: Union[datetime, str],
+        ignore_timeinfo: bool,
+        include_microseconds=False,
+        max=False,
     ):
         """Get an HDX date as a string from a datetime.datetime object.
 
         Args:
             date (Union[datetime, str]): Date as datetime or string
             ignore_timeinfo (bool): Ignore time and time zone of date. Defaults to True.
+            include_microseconds (bool): Include microsconds. Defaults to False.
 
         Returns:
             str: HDX date as a string
@@ -27,26 +31,34 @@ def get_hdx_date(
             timezone_handling = 3
 
         if isinstance(date, str):
-            date = parse_date(date, timezone_handling=timezone_handling)
+            date = parse_date(
+                date,
+                timezone_handling=timezone_handling,
+                include_microseconds=include_microseconds,
+            )
 
         if ignore_timeinfo:
             if max:
                 date = date.replace(
                     hour=23,
                     minute=59,
                     second=59,
-                    microsecond=0,
-                    tzinfo=None,
+                    microsecond=999999,
                 )
             else:
                 date = date.replace(
-                    hour=0, minute=0, second=0, microsecond=0, tzinfo=None
+                    hour=0,
+                    minute=0,
+                    second=0,
+                    microsecond=0,
                 )
         else:
-            date = date.astimezone(timezone.utc).replace(
-                microsecond=0, tzinfo=None
-            )
-        return date.isoformat()
+            date = date.astimezone(timezone.utc)
+        if include_microseconds:
+            timespec = "microseconds"
+        else:
+            timespec = "seconds"
+        return date.replace(tzinfo=None).isoformat(timespec=timespec)
 
     @staticmethod
     def get_reference_period_info(
@@ -93,8 +105,8 @@ def get_reference_period_info(
             result["startdate"] = startdate
             result["enddate"] = enddate
             if date_format is None:
-                startdate_str = startdate.isoformat()
-                enddate_str = enddate.isoformat()
+                startdate_str = startdate.isoformat(timespec="seconds")
+                enddate_str = enddate.isoformat(timespec="seconds")
             else:
                 startdate_str = startdate.strftime(date_format)
                 enddate_str = enddate.strftime(date_format)
@@ -123,14 +135,23 @@ def get_hdx_reference_period(
         Returns:
             str: HDX reference period
         """
-        startdate = cls.get_hdx_date(startdate, ignore_timeinfo)
+        startdate = cls.get_hdx_date(
+            startdate,
+            ignore_timeinfo=ignore_timeinfo,
+            include_microseconds=False,
+        )
         if ongoing:
             enddate = "*"
         else:
             if not enddate:
                 enddate = startdate
             else:
-                enddate = cls.get_hdx_date(enddate, ignore_timeinfo, max=True)
+                enddate = cls.get_hdx_date(
+                    enddate,
+                    ignore_timeinfo=ignore_timeinfo,
+                    include_microseconds=False,
+                    max=True,
+                )
         return f"[{startdate} TO {enddate}]"
 
     @classmethod

src/hdx/data/filestore_helper.py

@@ -100,5 +100,7 @@ def dataset_merge_filestore_resource(
         if resource.get_file_to_upload():
             resource["url"] = cls.temporary_url
         if data_updated:
-            resource["last_modified"] = datetime.utcnow().isoformat()
+            resource["last_modified"] = datetime.utcnow().isoformat(
+                timespec="microseconds"
+            )
             resource.data_updated = False

src/hdx/data/resource.py

@@ -370,7 +370,9 @@ def _resource_merge_hdx_update(
         """
         data_updated = kwargs.pop("data_updated", self.data_updated)
         if data_updated and not self.file_to_upload:
-            self.old_data["last_modified"] = datetime.utcnow().isoformat()
+            self.old_data["last_modified"] = datetime.utcnow().isoformat(
+                timespec="microseconds"
+            )
             self.data_updated = False
             # old_data will be merged into data in the next step
         self._merge_hdx_update(
@@ -769,20 +771,22 @@ def get_date_data_updated(self) -> datetime:
         Returns:
             datetime: Date resource data was updated
         """
-        return parse_date(self.data["last_modified"])
+        return parse_date(
+            self.data["last_modified"], include_microseconds=True
+        )
 
     def set_date_data_updated(
-        self, date: Union[datetime, str], ignore_timeinfo: bool = True
+        self, date: Union[datetime, str], ignore_timeinfo: bool = False
     ) -> None:
         """Set date resource data was updated
 
         Args:
             date (Union[datetime, str]): Date resource data was updated
-            ignore_timeinfo (bool): Ignore time and time zone of date. Defaults to True.
+            ignore_timeinfo (bool): Ignore time and time zone of date. Defaults to False.
 
         Returns:
             None
         """
         self.data["last_modified"] = DateHelper.get_hdx_date(
-            date, ignore_timeinfo
+            date, ignore_timeinfo=ignore_timeinfo, include_microseconds=True
         )

tests/hdx/data/test_resource.py

@@ -1019,8 +1019,17 @@ def test_date_data_updated(self, configuration, read):
         assert resource["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5"
         assert resource["name"] == "MyResource1"
         assert resource["package_id"] == "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d"
-        resource.set_date_data_updated("2020-03-02")
-        expected = "2020-03-02T00:00:00+00:00"
-        assert resource.get_date_data_updated().isoformat() == expected
-        resource.set_date_data_updated(parse_date("2020-03-02"))
-        assert resource.get_date_data_updated().isoformat() == expected
+        date = "2020-03-02 10:34:43.123456"
+        resource.set_date_data_updated(date)
+        expected = "2020-03-02T10:34:43.123456+00:00"
+        assert (
+            resource.get_date_data_updated().isoformat(timespec="microseconds")
+            == expected
+        )
+        resource.set_date_data_updated(
+            parse_date(date, include_microseconds=True)
+        )
+        assert (
+            resource.get_date_data_updated().isoformat(timespec="microseconds")
+            == expected
+        )