Merge pull request #71 from TUDelftGeodesy/70-metadata

Metadata orbit array

Author: Simon-van-Diepen

sarxarray/_io.py

@@ -12,6 +12,7 @@
 import xarray as xr
 
 from .conf import (
+    META_ARRAY_KEYS,
     META_FLOAT_KEYS,
     META_INT_KEYS,
     RE_PATTERNS_DORIS4,
@@ -360,7 +361,7 @@ def read_metadata(
     # if a key does not exists, a list will be created
     metadata = defaultdict(list)
     for file in files:
-        res = _parse_metadata(file, driver)
+        res = _parse_metadata(file, driver, ifg_file_name)
         for key, value in res.items():
             metadata[key].append(value)
 
@@ -370,7 +371,7 @@ def read_metadata(
     return metadata
 
 
-def _parse_metadata(file, driver, ifg_file_name="ifgs.res"):
+def _parse_metadata(file, driver, ifg_file_name):
     """Parse a single metadata file to a dictionary of strings."""
     # Select the appropriate patterns based on the driver
     if driver == "doris5":
@@ -379,6 +380,11 @@ def _parse_metadata(file, driver, ifg_file_name="ifgs.res"):
     elif driver == "doris4":
         patterns = RE_PATTERNS_DORIS4
         patterns_ifg = None
+    else:
+        raise NotImplementedError(
+            f"Driver '{driver}' is not implemented. "
+            "Supported drivers are: 'doris4', 'doris5'."
+        )
 
     # Open the file
     with open(file) as f:
@@ -387,11 +393,18 @@ def _parse_metadata(file, driver, ifg_file_name="ifgs.res"):
     # Read common metadata patterns
     results = {}
     for key, pattern in patterns.items():
-        match = re.search(pattern, content)
-        if match:
-            results[key] = match.group(1)
+        if key in META_ARRAY_KEYS.keys():  # multiple hits allowed
+            matches = re.findall(pattern, content)
+            if matches:
+                results[key] = matches
+            else:
+                results[key] = None
         else:
-            results[key] = None
+            match = re.search(pattern, content)
+            if match:
+                results[key] = match.group(1)
+            else:
+                results[key] = None
 
     # Doris5 has size information in ifgs.res file
     # Try to get the ifg size from ifgs.res next to slave.res, if it exists
@@ -400,12 +413,19 @@ def _parse_metadata(file, driver, ifg_file_name="ifgs.res"):
         if file_ifg.exists():
             with open(file_ifg) as f_ifg:
                 content_ifg = f_ifg.read()
-            for key, pattern in RE_PATTERNS_DORIS5_IFG.items():
-                match = re.search(pattern, content_ifg)
-                if match:
-                    results[key] = match.group(1)
-                else:
-                    results[key] = None
+                for key, pattern in RE_PATTERNS_DORIS5_IFG.items():
+                    if key in META_ARRAY_KEYS.keys():  # multiple hits allowed
+                        matches = re.findall(pattern, content_ifg)
+                        if matches:
+                            results[key] = matches
+                        else:
+                            results[key] = None
+                    else:
+                        match = re.search(pattern, content_ifg)
+                        if match:
+                            results[key] = match.group(1)
+                        else:
+                            results[key] = None
 
     return results
 
@@ -446,34 +466,51 @@ def _regulate_metadata(metadata, driver):
                 "Different types are found in the value list."
             )
 
-        # Only keep the unique values
-        if isinstance(metadata[key], list):
-            metadata[key] = set(value)
+        if key in META_ARRAY_KEYS.keys():  # need to regulate this one separately
+            regulated_arrays = []
+            for arr in metadata[key]:
+                regulated_array = np.zeros((len(arr), len(arr[0])))
+                for row in range(len(arr)):
+                    for col in range(len(arr[row])):
+                        regulated_array[row, col] = META_ARRAY_KEYS[key](arr[row][col])
+                regulated_arrays.append(np.copy(regulated_array))
 
-        # Unfold the single value set to strings
-        if len(metadata[key]) == 1:
-            metadata[key] = next(iter(metadata[key]))
+            metadata[key] = [
+                np.copy(regulated_array) for regulated_array in regulated_arrays
+            ]
+            if len(metadata[key]) == 1:
+                metadata[key] = metadata[key][0]
 
-        # if float, take the average unless std is larger than 1% of the mean
-        if key in META_FLOAT_KEYS:
-            # Convert to float
-            arr = np.array(value, dtype=np.float64)
-            if np.std(arr) / np.mean(arr) < 0.01:
-                metadata[key] = np.mean(arr).item()  # Convert to scalar
-            else:
-                raise ValueError(
-                    f"Inconsistency found in metadata key:  {key}. "
-                    "Standard deviation is larger than 1% of the mean."
-                )
-        if key in META_INT_KEYS:
-            if isinstance(metadata[key], str):
-                metadata[key] = int(metadata[key])
-            elif len(metadata[key]) > 1:  # set with multiple values
-                metadata[key] = set([int(v) for v in metadata[key]])
-
-        if key in ["number_of_lines", "number_of_pixels"]:
-            if isinstance(metadata[key], set):
-                warning_msg = f"Multiple values found in {key}: {metadata[key]}."
-                logger.warning(warning_msg)
+
+        else:
+            # Only keep the unique values
+            if isinstance(metadata[key], list):
+                metadata[key] = set(value)
+
+            # Unfold the single value set to strings
+            if len(metadata[key]) == 1:
+                metadata[key] = next(iter(metadata[key]))
+
+            # if float, take the average unless std is larger than 1% of the mean
+            if key in META_FLOAT_KEYS:
+                # Convert to float
+                arr = np.array(value, dtype=np.float64)
+                if np.std(arr) / np.mean(arr) < 0.01:
+                    metadata[key] = np.mean(arr).item()  # Convert to scalar
+                else:
+                    raise ValueError(
+                        f"Inconsistency found in metadata key:  {key}. "
+                        "Standard deviation is larger than 1% of the mean."
+                    )
+            if key in META_INT_KEYS:
+                if isinstance(metadata[key], str):
+                    metadata[key] = int(metadata[key])
+                elif len(metadata[key]) > 1:  # set with multiple values
+                    metadata[key] = set([int(v) for v in metadata[key]])
+
+            if key in ["number_of_lines", "number_of_pixels"]:
+                if isinstance(metadata[key], set):
+                    warning_msg = f"Multiple values found in {key}: {metadata[key]}."
+                    logger.warning(warning_msg)
 
     return metadata

sarxarray/conf.py

@@ -17,14 +17,14 @@
         r"Pulse_Repetition_Frequency \(computed, Hz\):\s+([\d\.E\+\-]+)"
     ),
     "total_azimuth_bandwidth": r"Total_azimuth_band_width \(Hz\):\s+([\d\.E\+\-]+)",
-    "range_time_first_pixel": (
+    "first_range_time": (
         r"Range_time_to_first_pixel \(2way\) \(ms\):\s+([\d\.E\+\-]+)"
     ),
     "range_sampling_rate": r"Range_sampling_rate \(computed, MHz\):\s+([\d\.E\+\-]+)",
     "total_range_bandwidth": r"Total_range_band_width \(MHz\):\s+([\d\.E\+\-]+)",
     "weighting_azimuth": r"Weighting_azimuth:\s+(.+)",
     "weighting_range": r"Weighting_range:\s+(.+)",
-    "first_pixel_azimuth_time": r"First_pixel_azimuth_time \(UTC\):\s+(.+)",
+    "first_azimuth_time": r"First_pixel_azimuth_time \(UTC\):\s+(.+)",
 }
 # Regular expressions for reading metadata from DORIS4 files
 RE_PATTERNS_DORIS5 = {
@@ -45,11 +45,11 @@
     "pulse_repetition_frequency": (
         r"Pulse_Repetition_Frequency \(computed, Hz\):\s+([\d\.E\+\-]+)"
     ),
-    "first_pixel_azimuth_time": r"First_pixel_azimuth_time \(UTC\):\s+(.+)",
+    "first_azimuth_time": r"First_pixel_azimuth_time \(UTC\):\s+(.+)",
     "azimuth_time_interval": r"Azimuth_time_interval \(s\):\s+([\d\.E\+\-]+)",
     "total_azimuth_bandwidth": r"Total_azimuth_band_width \(Hz\):\s+([\d\.E\+\-]+)",
     "weighting_azimuth": r"Weighting_azimuth:\s+(.+)",
-    "range_time_first_pixel": (
+    "first_range_time": (
         r"Range_time_to_first_pixel \(2way\) \(ms\):\s+([\d\.E\+\-]+)"
     ),
     "range_sampling_rate": r"Range_sampling_rate \(computed, MHz\):\s+([\d\.E\+\-]+)",
@@ -59,6 +59,18 @@
     "deramp": r"deramp:\s+([\d\.E\+\-]+)",
     "reramp": r"reramp:\s+([\d\.E\+\-]+)",
     "esd_correct": r"ESD_correct:\s+([\d\.E\+\-]+)",
+    "orbit_txyz": (
+        r"(\d+)\s+([-+]?\d+\.\d+(?:\.\d+)?)\s+([-+]?\d+"
+        r"\.\d+(?:\.\d+)?)\s+([-+]?\d+\.\d+(?:\.\d+)?)"
+    ),
+    "scene_centre_latitude": (
+        r"Scene_centre_latitude:"
+        r"\s+([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)"
+    ),
+    "scene_centre_longitude": (
+        r"Scene_centre_longitude:"
+        r"\s+([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)"
+    )
 }
 # Regular expressions for reading metadata from DORIS5 interferogram files
 RE_PATTERNS_DORIS5_IFG = {
@@ -71,14 +83,16 @@
     "wavelength",
     "pulse_repetition_frequency",
     "total_azimuth_bandwidth",
-    "range_time_first_pixel",
+    "first_range_time",
     "range_sampling_rate",
     "total_range_bandwidth",
     "range_pixel_spacing",  # from here DORIS5 only
     "azimuth_pixel_spacing",
     "radar_frequency",
     "pulse_repetition_frequency_raw",
     "azimuth_time_interval",
+    "scene_centre_latitude",
+    "scene_centre_longitude"
 ]
 # Integer keys in metadata. They are used to regulate the metadata read as strings
 META_INT_KEYS = [
@@ -88,8 +102,13 @@
     "number_of_pixels",
     "esd_correct",
 ]
+# Array keys in metadata and their format. Requires re.findall instead of re.match
+# Expects 2D arrays, and a callable variable type as value associated with each key
+META_ARRAY_KEYS = {
+    "orbit_txyz": float  # DORIS5 only
+}
 # Time formats for DORIS metadata
 TIME_FORMAT_DORIS4 = "%d-%b-%Y %H:%M:%S.%f"
 TIME_FORMAT_DORIS5 = "%Y-%b-%d %H:%M:%S.%f"
 # Time stamp key
-TIME_STAMP_KEY = "first_pixel_azimuth_time"
+TIME_STAMP_KEY = "first_azimuth_time"

tests/test_io.py

@@ -10,6 +10,7 @@
 import sarxarray
 from sarxarray._io import _calc_chunksize, _unpack_complex
 from sarxarray.conf import (
+    META_ARRAY_KEYS,
     META_FLOAT_KEYS,
     META_INT_KEYS,
     RE_PATTERNS_DORIS4,
@@ -185,17 +186,23 @@ def test_read_metadata_doris4(self, res_files_doris4):
                 assert isinstance(metadata[key], float)
             elif key in META_INT_KEYS:
                 assert isinstance(metadata[key], int)
-        assert metadata["first_pixel_azimuth_time"].shape[0] == 3
+        assert metadata["first_azimuth_time"].shape[0] == 3
 
     def test_read_metadata_doris5(self, res_files_doris5, caplog):
         with caplog.at_level(logging.WARNING):  # Last file has a wrong number of pixels
+            # also catches the multiple orbit warning
             metadata = sarxarray.read_metadata(res_files_doris5, driver="doris5")
         for key in RE_PATTERNS_DORIS5.keys():
             assert key in metadata
             if key in META_FLOAT_KEYS:
                 assert isinstance(metadata[key], float)
             elif key in META_INT_KEYS:
                 assert isinstance(metadata[key], int)
+            elif key in META_ARRAY_KEYS.keys():
+                assert isinstance(metadata[key][0], np.ndarray)
+                assert len(metadata[key]) == len(res_files_doris5)
+                assert len(metadata[key][0].shape) == 2
+                assert isinstance(metadata[key][0][0][0], META_ARRAY_KEYS[key])
         for key in RE_PATTERNS_DORIS5_IFG.keys():
             assert key in metadata
             if key in META_FLOAT_KEYS:
@@ -205,7 +212,7 @@ def test_read_metadata_doris5(self, res_files_doris5, caplog):
                     assert len(metadata[key]) == 2  # Two values for ifg sizes
                 else:
                     assert isinstance(metadata[key], int)
-        assert metadata["first_pixel_azimuth_time"].shape[0] == 3
+        assert metadata["first_azimuth_time"].shape[0] == 3
 
     def test_read_metadata_non_existent_driver(self, res_files_doris4):
         with pytest.raises(NotImplementedError):
@@ -219,7 +226,7 @@ def test_read_metadata_doris4_onefile(self, res_files_doris4):
                 assert isinstance(metadata[key], float)
             elif key in META_INT_KEYS:
                 assert isinstance(metadata[key], int)
-        assert np.isscalar(metadata["first_pixel_azimuth_time"])
+        assert np.isscalar(metadata["first_azimuth_time"])
 
     def test_read_metadata_doris5_onefile(self, res_files_doris5, caplog):
         metadata = sarxarray.read_metadata(res_files_doris5[0], driver="doris5")
@@ -229,10 +236,14 @@ def test_read_metadata_doris5_onefile(self, res_files_doris5, caplog):
                 assert isinstance(metadata[key], float)
             elif key in META_INT_KEYS:
                 assert isinstance(metadata[key], int)
+            elif key in META_ARRAY_KEYS.keys():
+                assert isinstance(metadata[key], np.ndarray)
+                assert len(metadata[key].shape) == 2
+                assert isinstance(metadata[key][0][0], META_ARRAY_KEYS[key])
         for key in RE_PATTERNS_DORIS5_IFG.keys():
             assert key in metadata
             if key in META_FLOAT_KEYS:
                 assert isinstance(metadata[key], float)
             elif key in META_INT_KEYS:
                 assert isinstance(metadata[key], int)
-        assert np.isscalar(metadata["first_pixel_azimuth_time"])
+        assert np.isscalar(metadata["first_azimuth_time"])