Load .jpk-qi-data metadata using jpk-reader-rs package

afmformats/formats/fmt_jpk/__init__.py

@@ -1,19 +1,18 @@
 import numpy as np
+import jpk_reader_rs as jpk_rs
 
 from ...errors import MissingMetaDataError
 from ...lazy_loader import LazyData
 from ...meta import LazyMetaValue
 
-from .jpk_reader import JPKReader
+from .jpk_reader import JPKReader, JPKQIMapReader
 
 
 __all__ = ["load_jpk"]
 
 
 def detect(path, return_modality=False):
-    """Check whether a file is a valid JPK data file
-
-    """
+    """Check whether a file is a valid JPK data file"""
     # The suffix is not checked, because that is done in
     # the wrapper class formats.AFMFormatRecipe.
     jpkr = JPKReader(path)
@@ -56,6 +55,62 @@ def load_jpk(path, callback=None, meta_override=None):
     jpkr = JPKReader(path)
     jpkr.set_metadata(meta_override)
 
+    dataset = []
+    # iterate over all datasets and add them
+    for index in range(len(jpkr)):
+        lazy_data = LazyData()
+        for column in [
+            "force",
+            "height (measured)",
+            "height (piezo)",
+            "segment",
+            "time",
+        ]:
+            lazy_data.set_lazy_loader(
+                column=column,
+                func=jpkr.get_data,
+                kwargs={"column": column, "index": index},
+            )
+        metadata = jpkr.get_metadata(index=index)
+        metadata["z range"] = LazyMetaValue(
+            lambda data: np.ptp(data["height (piezo)"]), lazy_data
+        )
+        dataset.append(
+            {
+                "data": lazy_data,
+                "metadata": metadata,
+            }
+        )
+        if callback:
+            callback((1 + index) / len(jpkr))
+    return dataset
+
+
+def load_jpk_qi_data(path, callback=None, meta_override=None):
+    """Loads JPK QMap data
+
+    These files are zip files containing java property files and
+    integer-encoded binary data. The property files include recipes
+    on how to convert the raw integer data to SI units.
+
+    Parameters
+    ----------
+    path: str or pathlib.Path
+        path to JPK data file
+    callback: callable
+        function for progress tracking; must accept a float in
+        [0, 1] as an argument.
+    meta_override: dict
+        if specified, contains key-value pairs of metadata that
+        are used when loading the files
+        (see :data:`afmformats.meta.META_FIELDS`)
+    """
+    if meta_override is None:
+        meta_override = {}
+
+    jpkr = JPKQIMapReader(path)
+    jpkr.set_metadata(meta_override)
+
     dataset = []
     # iterate over all datasets and add them
     for index in range(len(jpkr)):
@@ -99,7 +154,7 @@ def load_jpk(path, callback=None, meta_override=None):
 recipe_jpk_force_qi_data = {
     "descr": "binary QMap data",
     "detect": detect,
-    "loader": load_jpk,
+    "loader": load_jpk_qi_data,
     "maker": "JPK Instruments",
     "modalities": ["creep-compliance", "force-distance", "stress-relaxation"],
     "suffix": ".jpk-qi-data",

afmformats/formats/fmt_jpk/jpk_reader.py

@@ -2,9 +2,11 @@
 import copy
 import functools
 import zipfile
+import bisect
 
 import jprops
 import numpy as np
+import jpk_reader_rs as jpk_rs
 
 from ...errors import MissingMetaDataError
 from ... import meta
@@ -51,6 +53,22 @@ def get(zip_path):
 
 
 class JPKReader(object):
+    @staticmethod
+    def _files_sortkey(maxdigits):
+        """Create a function to use as the sort key for files"""
+        repstr = "{:0" + "{}".format(maxdigits) + "d}"
+        def key(x):
+            if x.count("/"):
+                xs = x.split("/")
+                for ii in range(len(xs)):
+                    if xs[ii].isnumeric():
+                        xs[ii] = repstr.format(int(xs[ii]))
+                return "/".join(xs)
+            else:
+                return x
+
+        return key
+
     def __init__(self, path):
         self.path = path
         self._user_metadata = {}
@@ -66,18 +84,7 @@ def files(self):
         arc = ArchiveCache.get(self.path)
         nlist = arc.namelist()
         maxdigits = int(np.ceil(np.log10(len(nlist)))) + 1
-        repstr = "{:0" + "{}".format(maxdigits) + "d}"
-
-        def sortkey(x):
-            if x.count("/"):
-                xs = x.split("/")
-                for ii in range(len(xs)):
-                    if xs[ii].isnumeric():
-                        xs[ii] = repstr.format(int(xs[ii]))
-                return "/".join(xs)
-            else:
-                return x
-
+        sortkey = self._files_sortkey(maxdigits)
         return sorted(nlist, key=sortkey)
 
     @property
@@ -481,3 +488,190 @@ def set_metadata(self, metadata):
         self._user_metadata.update(metadata)
         self.get_metadata.cache_clear()
         self._get_index_segment_properties.cache_clear()
+
+
+class JPKQIMapReader(JPKReader):
+    def __init__(self, path):
+        super().__init__(path)
+        self._reader = jpk_rs.QIMapReader(path)
+        self._files = self._reader.files()
+        self._metadata = self._reader.all_metadata()
+
+        shared_props = self._properties_shared
+        self._lcdinfo = {}
+        for key in shared_props:
+            parts = key.split(".", 2)
+            if len(parts) >= 3:
+                prefix = f"{parts[0]}.{parts[1]}."
+                var = parts[2]
+                if prefix not in self._lcdinfo:
+                    self._lcdinfo[prefix] = []
+                self._lcdinfo[prefix].append((var, shared_props[key]))
+
+        self._hierarchy = None
+        if self.files_contains("segments/"):
+            self._hierarchy = "single"
+        elif self.files_contains("index/"):
+            self._hierarchy = "indexed"
+
+    @property
+    @functools.lru_cache()
+    def files(self):
+        return self._files
+
+    @property
+    @functools.lru_cache()
+    def hierarchy(self):
+        """Format hierarchy ("single" or "indexed")"""
+        if self._hierarchy is None:
+            msg = "Cannot determine hierarchy: {}".format(self.path)
+            raise NotImplementedError(msg)
+        else:
+            return self._hierarchy
+
+    @property
+    @functools.lru_cache
+    def _properties_general(self):
+        """Return content of "header.properties"""
+        return self._metadata[("dataset", None, None)]
+
+    @property
+    @functools.lru_cache
+    def _properties_shared(self):
+        """Return content of "shared-data/header.properties"""
+        return self._metadata[("shared_data", None, None)]
+
+    @functools.lru_cache
+    def files_contains(self, path):
+        """Return whether the path is in files"""
+        maxdigits = int(np.ceil(np.log10(len(self.files)))) + 1
+        sortkey = self._files_sortkey(maxdigits)
+        sortpath = sortkey(path)
+        index = bisect.bisect_left(self.files, sortpath, key=sortkey)
+        return index < len(self.files) and self.files[index] == path
+
+    @functools.lru_cache()
+    def get_index_path(self, index):
+        """Return the path in the zip file for a specific curve index"""
+        enum = self.get_index_numbers()[index]
+        if self.hierarchy == "single":
+            path = ""
+        elif self.hierarchy == "indexed":
+            path = "index/{}/".format(enum)
+        else:
+            raise NotImplementedError("No rule to get path for hierarchy "
+                                      + "'{}'!".format(self.hierarchy))
+        if path and not self.files_contains(path):
+            raise IndexError("Cannot find path for index '{}' ".format(index)
+                             + " (enum '{}')!".format(enum))
+        return path
+
+    @functools.lru_cache()
+    def get_index_segment_path(self, index, segment):
+        """Return the path in the zip file for a specific index and segment"""
+        enum = self.get_index_numbers()[index]
+        if self.hierarchy == "single":
+            path = "segments/{}/".format(segment)
+        elif self.hierarchy == "indexed":
+            path = "index/{}/segments/{}/".format(enum, segment)
+        else:
+            raise NotImplementedError("No rule to get path for hierarchy "
+                                      + "'{}'!".format(self.hierarchy))
+        if not self.files_contains(path):
+            raise IndexError("Cannot find path for index '{}' ".format(index)
+                             + "(enum '{}')".format(enum))
+        return path
+
+
+    @functools.lru_cache
+    def _get_index_segment_properties(self, index, segment):
+        """Return properties from a specific index and segment
+
+        Parameters
+        ----------
+        index: int
+            Curve index; For "single" hierarchy files, this should be 0.
+        segment: int or None
+            If None, then no segment-specific properties (e.g.
+            approach or retract) are returned.
+        """
+        # 1. Properties of index
+        prop = self._metadata[("index", index, None)]
+
+        # 2. Properties of segment (if applicable)
+        if segment is not None:
+            segment_prop = self._metadata[("segment", index, segment)]
+            prop.update(segment_prop)
+
+        # 3. Substitute shared properties
+        # Generate lists of keys.
+        proplist = list(prop.keys())
+        # Loop through the segment data and search for lcd-info tags
+        for key in proplist:
+            # Get line channel data
+            if ".*" in key:
+                # Replace the lcd-info tag by the values in the shared
+                # properties file:
+                # 0, 1, 2, 3, etc.
+                pindex = prop[key]
+                # lcd-info, force-segment-header-info
+                mediator = ".".join(key.split(".")[-2:-1])
+                # channel.vDeflection, force-segment-header
+                headkey = key.rsplit(".", 2)[0]
+                # append a "." here to make sure
+                # not to confuse "1" with "10".
+                startid = "{}.{}.".format(mediator, pindex)
+                if startid in self._lcdinfo:
+                    for var, value in self._lcdinfo[startid]:
+                        prop[".".join([headkey, var])] = value
+
+        # 4. Update with general properties
+        # (for "single" hierarchy, this coincides with index properties)
+        prop.update(self._properties_general)
+
+        # 5. Try to convert numbers to floats and remove NaNs
+        for p in list(prop.keys()):
+            try:
+                prop[p] = float(prop[p])
+            except BaseException:
+                pass
+            else:
+                if np.isnan(prop[p]):
+                    prop.pop(p)
+
+        # 6. sneakily insert spring constant and sensitivity into the property
+        #    lists. This is manipulation of metadata at the lowest possible
+        #    level. We need it in :func:`jpk_data.load_dat_unit` as well
+        #    as in :func:`.get_metadata`.
+        prmet = jpk_meta.get_primary_meta_recipe()
+        for key, base_slot, unit in [
+                ("spring constant", "distance", "N"),
+                ("sensitivity", "volts", "m")]:
+            if key in self._user_metadata:
+                for opt_mult in prmet[key]:
+                    # channel.vDeflection.conversion-set.conversion.
+                    # distance.scaling.multiplier
+                    prop[opt_mult] = self._user_metadata[key]
+                    # channel.vDeflection.conversion-set.conversion.
+                    # distance.scaling.offset
+                    opt_off = opt_mult.rsplit(".", 1)[0] + ".offset"
+                    prop[opt_off] = 0
+                    # channel.vDeflection.conversion-set.conversion.
+                    # distance.scaling.unit
+                    opt_unit = opt_mult.rsplit(".", 1)[0] + ".unit"
+                    prop[opt_unit] = unit
+                    # channel.vDeflection.conversion-set.conversion.
+                    # distance.base-calibration-slot
+                    opt_slot = (opt_mult.rsplit(".", 2)[0]
+                                + ".base-calibration-slot")
+                    prop[opt_slot] = base_slot
+        return prop
+
+    def set_metadata(self, metadata):
+        """Override internal metadata
+
+        This has a direct effect on :func:`.get_metadata`.
+        """
+        self._user_metadata.clear()
+        self._user_metadata.update(metadata)
+        self.get_metadata.cache_clear()

afmformats/meta.py

@@ -404,6 +404,12 @@ def parse_time(value):
     else:
         ss0 = ss
         ss1 = ""
+
+    # strip escape characters.
+    # See https://github.com/AFM-analysis/afmformats/issues/43
+    hh = hh.strip("\\")
+    mm = mm.strip("\\")
+
     newvalue = ":".join(["{:02d}".format(int(hh)),
                          "{:02d}".format(int(mm)),
                          "{:02d}".format(int(ss0)) + ss1