Merge pull request #111 from HydrologicEngineeringCenter/preCompression

Enable User to write a precompressed grid to dss

Author: oskarhurst

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "hec-dss-python"
-version = "0.1.28"
+version = "0.1.29"
 description = "Python wrapper for the HEC-DSS file database C library."
 authors = ["Hydrologic Engineering Center"]
 license = "MIT"

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = hecdss
-version = 0.1.28
+version = 0.1.29
 author = Hydrologic Engineering Center
 author_email =hec.dss@usace.army.mil
 description = Python wrapper for the HEC-DSS file database C library.

src/hecdss/download_hecdss.py

@@ -36,7 +36,7 @@ def download_and_unzip(url, zip_file, destination_dir):
         print(f"Failed to download zip file. Status code: {response.status_code}")
 
 base_url = "https://www.hec.usace.army.mil/nexus/repository/maven-public/mil/army/usace/hec/hecdss/"
-version = "7-IW-4"
+version = "7-JA-4"
 
 destination_dir = Path(__file__).parent.joinpath("lib")
 zip_url = f"{base_url}{version}-win-x86_64/hecdss-{version}-win-x86_64.zip"

src/hecdss/hecdss.py

@@ -679,6 +679,24 @@ def put(self, container) -> int:
 
         return status
 
+
+    def writePrecompressedGrid(self, gd, compressedData, CompressionSize):
+        """
+        puts pre-compressed gridded data into the DSS file
+
+        Args
+            compressedData (bytes): Compressed data.
+            CompressionSize (int): Size of the compressed data.
+        Returns:
+            int: 0 if successful, -1 otherwise.
+        """
+
+        if compressedData and CompressionSize > 0:
+            status = self._native.hec_dss_gridStore(gd, compressedData, CompressionSize)
+            self._catalog = None
+            return status
+        return -1
+
     def delete(self, pathname: str, allrecords: bool = False, startdatetime=None, enddatetime=None) -> int:
         """deletes a record from the DSS file
         Args:

src/hecdss/native.py

@@ -332,6 +332,8 @@ def hec_dss_gridRetrieve(self, pathname: str,
     def hec_dss_gridStore(
             self,
             gd,
+            compressedData=None,
+            compressionSize=0,
     ):
         self.dll.hec_dss_pdStore.restype = c_int
         self.dll.hec_dss_pdStore.argtypes = [
@@ -348,6 +350,7 @@ def hec_dss_gridStore(
             ctypes.c_int,  # timeZoneRawOffset
             ctypes.c_int,  # isInterval
             ctypes.c_int,  # isTimeStamped
+            ctypes.c_int,  # compressionSize
             ctypes.c_char_p,  # dataUnits
             ctypes.c_char_p,  # dataSource
             ctypes.c_char_p,  # srsName
@@ -378,6 +381,7 @@ def hec_dss_gridStore(
         c_timeZoneRawOffset = c_int(gd.timeZoneRawOffset)
         c_isInterval = c_int(gd.isInterval)
         c_isTimeStamped = c_int(gd.isTimeStamped)
+        c_compressionSize = c_int(compressionSize)  # default compression
 
         c_dataUnits = c_char_p(gd.dataUnits.encode("utf-8"))
         c_dataSource = c_char_p(gd.dataSource.encode("utf-8"))
@@ -397,15 +401,19 @@ def hec_dss_gridStore(
         c_numberEqualOrExceedingRangeLimit = (c_int * len(gd.numberEqualOrExceedingRangeLimit))(
             *gd.numberEqualOrExceedingRangeLimit)
 
-        arr = gd.data.astype('float32', copy=False)
-        c_data = arr.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
+        if compressedData is not None and compressionSize:
+            # Treat compressed data as raw bytes, not float32
+            c_data = ctypes.cast(compressedData, ctypes.POINTER(ctypes.c_float))
+        else:
+            arr = gd.data.astype('float32', copy=False)
+            c_data = arr.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
 
         return self.dll.hec_dss_gridStore(self.handle, c_pathname, c_gridType, c_dataType,
                                           c_lowerLeftCellX, c_lowerLeftCellY,
                                           c_numberOfCellsX, c_numberOfCellsY,
                                           c_numberOfRanges, c_srsDefinitionType,
                                           c_timeZoneRawOffset, c_isInterval,
-                                          c_isTimeStamped,
+                                          c_isTimeStamped, c_compressionSize,
                                           c_dataUnits, c_dataSource,
                                           c_srsName, c_srsDefinition, c_timeZoneID,
                                           c_cellSize, c_xCoordOfGridCellZero,

tests/test_gridded_data.py

@@ -86,6 +86,44 @@ def test_gridded_data_read_store_read(self):
             assert (
                     gd.dataUnits == gd2.dataUnits), f"gd2.dataUnits is not equal to {gd.dataUnits}. gd2.dataUnits is {gd2.dataUnits}"
 
+    def test_gridded_data_write_precompressed(self):
+        """
+        Test writing precompressed gridded data using zlib deflate.
+        Reads existing grid data, compresses it, writes using writePrecompressedGrid, and compares.
+        """
+        import zlib
+
+        # Read existing gridded data
+        original_path = "/grid/EAU GALLA RIVER/SNOW MELT/02FEB2020:0600/03FEB2020:0600/SHG-SNODAS/"
+        file = self.test_files.get_copy("grid-example.dss")
+
+        with HecDss(file) as dss:
+            # Read the original grid
+            gd_original = dss.get(original_path)
+
+            # Compress the data using zlib deflate
+            raw_bytes = gd_original.data.astype(np.float32).tobytes()
+            compressed_data = zlib.compress(raw_bytes)
+            compression_size = len(compressed_data)
+
+            # Create a new GriddedData object with metadata from original
+            # but pointing to a new path
+            new_path = "/grid/EAU GALLA RIVER/SNOW MELT/02FEB2020:0600/03FEB2020:0600/SHG-SNODAS-COMPRESSED/"
+            gd_original.id = new_path
+
+
+            # Write the precompressed grid
+            status = dss.writePrecompressedGrid(gd_original, compressed_data, compression_size)
+
+            # Read back the compressed grid
+            gd_readback = dss.get(new_path)
+
+            # Compare the two grids
+            assert status == 0, f"writePrecompressedGrid status should be 0, is {status}"
+            assert np.array_equal(gd_original.data, gd_readback.data), "Data from original and compressed grid do not match"
+            assert gd_original.numberOfCellsX == gd_readback.numberOfCellsX, "numberOfCellsX mismatch"
+            assert gd_original.numberOfCellsY == gd_readback.numberOfCellsY, "numberOfCellsY mismatch"
+            assert gd_original.dataUnits == gd_readback.dataUnits, "dataUnits mismatch"
 
 
 if __name__ == "__main__":