fix: multiple smaller issues related to hashing of remote data

Author: paulmueller

CHANGELOG

@@ -1,4 +1,7 @@
 0.71.4
+ - fix: dataset `identifier` is session-unique for `RTDC_HTTP`
+ - fix: dataset `hash` must be globally unique for `RTDC_HTTP` and `RTDC_S3`
+ - fix: strip quotes from ETag in `S3File`
  - enh: add unified `logger` property for all instances of `RTDCBase`
 0.71.3
  - enh: disable HDF5 file locking for basic reading operations

dclab/rtdc_dataset/core.py

@@ -314,6 +314,7 @@ def _get_basin_feature_data(
                     #  especially when considering networking issues.
                     if feat in bn.features:
                         data = bn.get_feature_data(feat)
+                        self.logger.info(f"Feature '{feat}' found in '{bn}'")
                         # The data are available, we may abort the search.
                         break
                 except feat_basin.BasinIdentifierMismatchError:

dclab/rtdc_dataset/fmt_hdf5/base.py

@@ -65,7 +65,7 @@ def __init__(self,
 
         self._hash = None
 
-        #: Path to the experimental HDF5 (.rtdc) file
+        #: Path to the measurement HDF5 (.rtdc) file
         self.path = h5path
 
         # Increase the read cache (which defaults to 1MiB), since
@@ -208,9 +208,17 @@ def parse_config(h5path):
     def hash(self):
         """Hash value based on file name and content"""
         if self._hash is None:
-            tohash = [self.path.name,
-                      # Hash a maximum of ~1MB of the hdf5 file
-                      hashfile(self.path, blocksize=65536, count=20)]
+            tohash = []
+            if isinstance(self.path, pathlib.Path):
+                # actual path on file system
+                tohash.append(self.path.name)
+            elif isinstance(self.path, str):
+                # remote location (when `hash` not defined in subclass)
+                tohash.append(self.path)
+            tohash.append(
+                # Hash a maximum of ~1MB of the hdf5 file
+                hashfile(self.path, blocksize=65536, count=20)
+            )
             self._hash = hashobj(tohash)
         return self._hash
 

dclab/rtdc_dataset/fmt_http.py

@@ -40,15 +40,6 @@ def __init__(self,
                 f"Package `requests` required for loading http data '{url}'!")
 
         self._fhttp = HTTPFile(url)
-        if kwargs.get("identifier") is None:
-            if self._fhttp.etag is not None:
-                # Set the HTTP ETag as the identifier, it doesn't get
-                # more unique than that!
-                kwargs["identifier"] = self._fhttp.etag
-            else:
-                # Compute a hash of the first data chunk
-                kwargs["identifier"] = hashlib.md5(
-                    self._fhttp.get_chunk(0)).hexdigest()
 
         # Initialize the HDF5 dataset
         super(RTDC_HTTP, self).__init__(
@@ -59,6 +50,16 @@ def __init__(self,
         #: URL to the file
         self.path = url
 
+    @property
+    def hash(self):
+        if self._fhttp.etag is not None:
+            # Set the HTTP ETag as the hash, it doesn't get
+            # more unique than that!
+            return self._fhttp.etag
+        else:
+            # Compute a hash of the first data chunk
+            return hashlib.md5(self._fhttp.get_chunk(0)).hexdigest()
+
     def close(self):
         super(RTDC_HTTP, self).close()
         self._fhttp.close()

dclab/rtdc_dataset/fmt_s3.py

@@ -1,4 +1,5 @@
 import functools
+import hashlib
 # import multiprocessing BaseManager here, because there is some kind
 # of circular dependency issue with s3transfer.compat and multiprocessing.
 from multiprocessing.managers import BaseManager  # noqa: F401
@@ -130,7 +131,7 @@ def __init__(self,
     def _parse_header(self):
         if self._len is None:
             self._len = self.s3_object.content_length
-            self._etag = self.s3_object.e_tag
+            self._etag = self.s3_object.e_tag.strip("'").strip('"')
 
     def close(self):
         super(S3File, self).close()
@@ -207,6 +208,16 @@ def __init__(self,
         #: URL the object on S3
         self.path = self._s3file.url
 
+    @property
+    def hash(self):
+        if self._s3file.etag is not None:
+            # Set the HTTP ETag as the hash, it doesn't get
+            # more unique than that!
+            return self._s3file.etag
+        else:
+            # Compute a hash of the first data chunk
+            return hashlib.md5(self._s3file.get_chunk(0)).hexdigest()
+
     def close(self):
         super(RTDC_S3, self).close()
         self._s3file.close()

tests/test_rtdc_fmt_http.py

@@ -34,12 +34,12 @@ def test_cache_features():
         assert t2 - t1 < t1 - t0
 
 
-def test_identifier():
+def test_hash():
     with RTDC_HTTP(s3_url) as ds:
         # This is the HTTP ETag (https://en.wikipedia.org/wiki/HTTP_ETag)
         # given to this resource by the object store. If the file is
         # re-uploaded, the ETag may change and this test will fail.
-        assert ds.identifier == "6dd392feb1aeda7cfb73b4ec76c1fe7c"
+        assert ds.hash == "6dd392feb1aeda7cfb73b4ec76c1fe7c"
 
 
 @pytest.mark.parametrize("netloc", [

tests/test_rtdc_fmt_s3.py

@@ -35,6 +35,14 @@ def test_cache_features():
         assert t2 - t1 < t1 - t0
 
 
+def test_hash():
+    with RTDC_S3(s3_url) as ds:
+        # This is the HTTP ETag (https://en.wikipedia.org/wiki/HTTP_ETag)
+        # given to this resource by the object store. If the file is
+        # re-uploaded, the ETag may change and this test will fail.
+        assert ds.hash == "6dd392feb1aeda7cfb73b4ec76c1fe7c"
+
+
 @pytest.mark.parametrize("url, avail", [
     ("https://objectstore.hpccloud.mpcdf.mpg.de/"
      "circle-5a7a053d-55fb-4f99-960c-f478d0bd418f/"