Hash files on upload

Author: mcarans

requirements.txt

@@ -22,7 +22,7 @@ cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
     # via frictionless
-charset-normalizer==3.4.2
+charset-normalizer==3.4.3
     # via requests
 ckanapi==4.8
     # via hdx-python-api (pyproject.toml)
@@ -32,7 +32,7 @@ click==8.2.1
     #   typer
 colorama==0.4.6
     # via mkdocs-material
-coverage==7.10.2
+coverage==7.10.3
     # via pytest-cov
 defopt==7.0.0
     # via hdx-python-api (pyproject.toml)
@@ -72,7 +72,7 @@ hdx-python-utilities==3.9.0
     #   hdx-python-country
 humanize==4.12.3
     # via frictionless
-identify==2.6.12
+identify==2.6.13
     # via pre-commit
 idna==3.10
     # via
@@ -115,9 +115,9 @@ markdown==3.8.2
     #   mkdocs
     #   mkdocs-material
     #   pymdown-extensions
-markdown-it-py==3.0.0
+markdown-it-py==4.0.0
     # via rich
-marko==2.1.4
+marko==2.2.0
     # via frictionless
 markupsafe==3.0.2
     # via
@@ -175,7 +175,7 @@ ply==3.11
     #   libhxl
 pockets==0.9.1
     # via sphinxcontrib-napoleon
-pre-commit==4.2.0
+pre-commit==4.3.0
     # via hdx-python-api (pyproject.toml)
 pyasn1==0.6.1
     # via
@@ -253,7 +253,7 @@ rfc3986==2.0.0
     # via frictionless
 rich==14.1.0
     # via typer
-rpds-py==0.26.0
+rpds-py==0.27.0
     # via
     #   jsonschema
     #   referencing

src/hdx/data/resource.py

@@ -1,5 +1,6 @@
 """Resource class containing all logic for creating, checking, and updating resources."""
 
+import hashlib
 import logging
 import warnings
 from datetime import datetime
@@ -358,15 +359,17 @@ def check_required_fields(self, ignore_fields: ListTuple[str] = tuple()) -> None
         self.check_url_filetoupload()
         self._check_required_fields("resource", ignore_fields)
 
-    def _get_files(self) -> Dict:
-        """Return the files parameter for CKANAPI
+    def _get_hash(self) -> str:
+        """Return the hash of file to upload
 
         Returns:
-            Dict: files parameter for CKANAPI
+            str: Hash of file to upload
         """
-        if self.file_to_upload is None:
-            return {}
-        return {"upload": self.file_to_upload}
+        md5 = hashlib.md5()
+        f = open(self.file_to_upload, "rb")
+        while chunk := f.read(4096):
+            md5.update(chunk)
+        return md5.hexdigest()
 
     def _resource_merge_hdx_update(
         self,
@@ -384,14 +387,21 @@ def _resource_merge_hdx_update(
             None
         """
         data_updated = kwargs.pop("data_updated", self.data_updated)
-        if data_updated and not self.file_to_upload:
+        files = {}
+        if self.file_to_upload:
+            hash = self._get_hash()
+            if hash != self.data.get("hash"):  # update file if hash has changed
+                files["upload"] = self.file_to_upload
+                self.old_data["hash"] = hash
+        elif data_updated:
             # Should not output timezone info here
             self.old_data["last_modified"] = now_utc_notz().isoformat(
                 timespec="microseconds"
             )
             self.data_updated = False
-            # old_data will be merged into data in the next step
-        self._merge_hdx_update("resource", "id", self._get_files(), True, **kwargs)
+
+        # old_data will be merged into data in the next step
+        self._merge_hdx_update("resource", "id", files, True, **kwargs)
 
     def update_in_hdx(self, **kwargs: Any) -> None:
         """Check if resource exists in HDX and if so, update it. To indicate
@@ -441,7 +451,11 @@ def create_in_hdx(self, **kwargs: Any) -> None:
                 del self.data["url"]
             self._resource_merge_hdx_update(**kwargs)
         else:
-            self._save_to_hdx("create", "name", self._get_files(), True)
+            files = {}
+            if self.file_to_upload:
+                files["upload"] = self.file_to_upload
+                self.data["hash"] = self._get_hash()
+            self._save_to_hdx("create", "name", files, True)
 
     def delete_from_hdx(self) -> None:
         """Deletes a resource from HDX