Fix marking resource data updated when using create_in_hdx and update_in_hdx of dataset

Author: Mike

src/hdx/data/dataset.py

@@ -290,6 +290,8 @@ def add_update_resource(
                 updated_resource.set_file_to_upload(
                     resource.get_file_to_upload()
                 )
+            if resource.is_data_updated():
+                updated_resource.mark_data_updated()
 
     def add_update_resources(
         self,
@@ -332,6 +334,8 @@ def add_update_resources(
                 updated_resource.set_file_to_upload(
                     resource.get_file_to_upload()
                 )
+            if resource.is_data_updated():
+                updated_resource.mark_data_updated()
         for resource_index in updated_resource_no_matches:
             resource = resource_objects[resource_index]
             resource.check_url_filetoupload()
@@ -470,7 +474,9 @@ def _dataset_create_resources(self) -> None:
 
         if "resources" in self.data:
             self.old_data["resources"] = self._copy_hdxobjects(
-                self.resources, res_module.Resource, "file_to_upload"
+                self.resources,
+                res_module.Resource,
+                ("file_to_upload", "data_updated"),
             )
             self.init_resources()
             self.separate_resources()

src/hdx/data/filestore_helper.py

@@ -1,5 +1,6 @@
 """Helper to the Dataset class for handling resources with filestores.
 """
+from datetime import datetime
 from typing import TYPE_CHECKING, Any, Dict
 
 from hdx.utilities.dictandlist import merge_two_dictionaries
@@ -91,9 +92,13 @@ def dataset_merge_filestore_resource(
         if file_to_upload:
             resource.set_file_to_upload(file_to_upload)
             filestore_resources[resource_index] = file_to_upload
+        data_updated = updated_resource.is_data_updated()
         merge_two_dictionaries(resource, updated_resource)
         cls.resource_check_required_fields(
             resource, check_upload=True, **kwargs
         )
         if resource.get_file_to_upload():
             resource["url"] = cls.temporary_url
+        if data_updated:
+            resource["last_modified"] = datetime.utcnow().isoformat()
+            resource.data_updated = False

src/hdx/data/hdxobject.py

@@ -621,14 +621,14 @@ def _copy_hdxobjects(
         self,
         hdxobjects: ListTuple["HDXObject"],
         hdxobjectclass: type,
-        attribute_to_copy: Optional[str] = None,
+        attributes_to_copy: ListTuple[str] = (),
     ) -> List["HDXObject"]:
         """Helper function to make a deep copy of a supplied list of HDX objects
 
         Args:
             hdxobjects (ListTuple[HDXObject]): list of HDX objects to copy
             hdxobjectclass (type): Type of the HDX Objects to be copied
-            attribute_to_copy (Optional[str]): An attribute to copy over from the HDX object. Defaults to None.
+            attributes_to_copy (ListTuple[str]): Attributes to copy over from the HDX object. Defaults to ().
 
         Returns:
             List[HDXObject]: Deep copy of list of HDX objects
@@ -639,9 +639,9 @@ def _copy_hdxobjects(
             newhdxobject = hdxobjectclass(
                 newhdxobjectdata, configuration=self.configuration
             )
-            if attribute_to_copy:
-                value = getattr(hdxobject, attribute_to_copy)
-                setattr(newhdxobject, attribute_to_copy, value)
+            for attribute in attributes_to_copy:
+                value = getattr(hdxobject, attribute)
+                setattr(newhdxobject, attribute, value)
             newhdxobjects.append(newhdxobject)
         return newhdxobjects
 

tests/hdx/conftest.py

@@ -1,4 +1,5 @@
 """Global fixtures"""
+import re
 import smtplib
 from os.path import join
 
@@ -1548,3 +1549,8 @@ class MockSMTP(MockSMTPBase):
     monkeypatch.setattr(smtplib, "SMTP_SSL", MockSMTPSSL)
     monkeypatch.setattr(smtplib, "LMTP", MockLMTP)
     monkeypatch.setattr(smtplib, "SMTP", MockSMTP)
+
+
+@pytest.fixture(scope="session")
+def date_pattern():
+    return re.compile(r"[12]\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\d\d\d\d")

tests/hdx/data/test_dataset_core.py

@@ -632,7 +632,7 @@ def test_create_in_hdx(self, configuration, post_create):
         assert len(dataset.resources) == 3
         # Dataset creates that end up updating are in the test below
 
-    def test_update_in_hdx(self, configuration, post_update):
+    def test_update_in_hdx(self, configuration, post_update, date_pattern):
         dataset = Dataset()
         dataset["id"] = "NOTEXIST"
         with pytest.raises(HDXError):
@@ -707,10 +707,15 @@ def test_update_in_hdx(self, configuration, post_update):
         dataset = Dataset(datasetdata)
         resourcesdata = copy.deepcopy(resources_data)
         resource = Resource(resourcesdata[0])
+        resource.mark_data_updated()
         dataset.add_update_resources([resource, resource])
         dataset.create_in_hdx()
         assert dataset["state"] == "active"
         assert len(dataset.resources) == 3
+        resource = dataset.get_resource()
+        assert resource.is_data_updated() is False
+        match = date_pattern.search(resource["last_modified"])
+        assert match
 
         dataset = Dataset(datasetdata)
         resourcesdata = copy.deepcopy(resources_data)

tests/hdx/data/test_resource.py

@@ -2,7 +2,6 @@
 import copy
 import json
 import os
-import re
 from datetime import datetime, timezone
 from os import remove
 from os.path import basename, join
@@ -329,10 +328,6 @@ class TestResource:
 
     datastore = None
 
-    @pytest.fixture(scope="class")
-    def date_pattern(self):
-        return re.compile(r"[12]\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\d\d\d\d")
-
     @pytest.fixture(scope="class")
     def static_yaml(self):
         return join("tests", "fixtures", "config", "hdx_resource_static.yaml")

tests/hdx/data/test_update_dataset_resources.py

@@ -85,7 +85,7 @@ def test_dataset_update_resources(
     ):
         dataset.old_data = new_dataset.data
         dataset.old_data["resources"] = new_dataset._copy_hdxobjects(
-            new_dataset.resources, Resource, "file_to_upload"
+            new_dataset.resources, Resource, ("file_to_upload", "data_updated")
         )
         (
             resources_to_delete,