Add LZ4 and LERC compression codecs for geotiff (#1063)

* Add LZ4 compression support for GeoTIFF module (#1051)

Add CPU LZ4 codec using python-lz4 (lz4.frame format) for TIFF
compression tag 50004. GPU nvCOMP acceleration deferred to a follow-up
since GDAL uses lz4-frame format which needs bridging to nvCOMP's raw
LZ4 format -- the CPU fallback path handles GPU reads/writes correctly
in the meantime.

Changes:
- _compression.py: lz4_decompress/lz4_compress functions, COMPRESSION_LZ4
  constant, wired into decompress/compress dispatchers
- _writer.py: 'lz4' key in _compression_tag mapping
- tests/test_lz4.py: 13 tests covering codec roundtrips, write-read
  roundtrips (tiled, stripped, float32, predictor), public API, and
  error handling when lz4 is not installed

* Add LERC compression for controlled-error lossy raster encoding (#1052)

Adds support for LERC (Limited Error Raster Compression, TIFF tag 34887)
to the geotiff module. LERC is CPU-only and wraps Esri's C library via
the `lerc` Python package. The key feature is the max_z_error parameter
which guarantees a maximum per-pixel encoding error (0 = lossless).

Changes:
- _compression.py: lerc_compress/lerc_decompress functions, COMPRESSION_LERC
  constant, dispatch entries in compress() and decompress()
- _writer.py: LERC in compression tag map, strip writer, tile writer
- _gpu_decode.py: explicit CPU fallback for LERC in both decode and encode
  paths (no GPU LERC library exists)
- tests/test_lerc.py: codec roundtrips (float32, uint8, uint16), lossy
  tolerance check, write-read roundtrips (tiled, stripped, public API),
  availability flag tests

Author: brendancol

xrspatial/geotiff/_compression.py

@@ -790,6 +790,143 @@ def jpeg2000_compress(data: bytes, width: int, height: int,
             os.unlink(tmp)
 
 
+# -- LERC codec (via lerc) ----------------------------------------------------
+
+LERC_AVAILABLE = False
+try:
+    import lerc as _lerc
+    LERC_AVAILABLE = True
+except ImportError:
+    _lerc = None
+
+
+def lerc_decompress(data: bytes, width: int = 0, height: int = 0,
+                    samples: int = 1) -> bytes:
+    """Decompress LERC data. Requires the ``lerc`` package."""
+    if not LERC_AVAILABLE:
+        raise ImportError(
+            "lerc is required to read LERC-compressed TIFFs. "
+            "Install it with: pip install lerc")
+    result = _lerc.decode(data)
+    # lerc.decode returns (result_code, data_array, valid_mask, ...)
+    if result[0] != 0:
+        raise RuntimeError(f"LERC decode failed with error code {result[0]}")
+    arr = result[1]
+    return arr.tobytes()
+
+
+def lerc_compress(data: bytes, width: int, height: int,
+                  samples: int = 1, dtype: np.dtype = np.dtype('float32'),
+                  max_z_error: float = 0.0) -> bytes:
+    """Compress raw pixel data with LERC. Requires the ``lerc`` package.
+
+    Parameters
+    ----------
+    max_z_error : float
+        Maximum encoding error per pixel. 0 = lossless.
+    """
+    if not LERC_AVAILABLE:
+        raise ImportError(
+            "lerc is required to write LERC-compressed TIFFs. "
+            "Install it with: pip install lerc")
+    if samples == 1:
+        arr = np.frombuffer(data, dtype=dtype).reshape(height, width)
+    else:
+        arr = np.frombuffer(data, dtype=dtype).reshape(height, width, samples)
+    n_values_per_pixel = samples
+    # lerc.encode(npArr, nValuesPerPixel, bHasMask, npValidMask,
+    #             maxZErr, nBytesHint)
+    # nBytesHint=1 triggers actual encoding (0 = compute size only)
+    result = _lerc.encode(arr, n_values_per_pixel, False, None,
+                          max_z_error, 1)
+    if result[0] != 0:
+        raise RuntimeError(f"LERC encode failed with error code {result[0]}")
+    # result is (error_code, nBytesWritten, ctypes_buffer)
+    return bytes(result[2])
+
+
+# -- LZ4 codec (via python-lz4) -----------------------------------------------
+
+LZ4_AVAILABLE = False
+try:
+    import lz4.frame as _lz4
+    LZ4_AVAILABLE = True
+except ImportError:
+    _lz4 = None
+
+
+def lz4_decompress(data: bytes) -> bytes:
+    """Decompress LZ4 frame data. Requires the ``lz4`` package."""
+    if not LZ4_AVAILABLE:
+        raise ImportError(
+            "lz4 is required to read LZ4-compressed TIFFs. "
+            "Install it with: pip install lz4")
+    return _lz4.decompress(data)
+
+
+def lz4_compress(data: bytes, level: int = 0) -> bytes:
+    """Compress data with LZ4 frame format. Requires the ``lz4`` package."""
+    if not LZ4_AVAILABLE:
+        raise ImportError(
+            "lz4 is required to write LZ4-compressed TIFFs. "
+            "Install it with: pip install lz4")
+    return _lz4.compress(data, compression_level=level)
+
+
+# -- LERC codec (via lerc) ----------------------------------------------------
+
+LERC_AVAILABLE = False
+try:
+    import lerc as _lerc
+    LERC_AVAILABLE = True
+except ImportError:
+    _lerc = None
+
+
+def lerc_decompress(data: bytes, width: int = 0, height: int = 0,
+                    samples: int = 1) -> bytes:
+    """Decompress LERC data. Requires the ``lerc`` package."""
+    if not LERC_AVAILABLE:
+        raise ImportError(
+            "lerc is required to read LERC-compressed TIFFs. "
+            "Install it with: pip install lerc")
+    result = _lerc.decode(data)
+    # lerc.decode returns (result_code, data_array, valid_mask, ...)
+    if result[0] != 0:
+        raise RuntimeError(f"LERC decode failed with error code {result[0]}")
+    arr = result[1]
+    return arr.tobytes()
+
+
+def lerc_compress(data: bytes, width: int, height: int,
+                  samples: int = 1, dtype: np.dtype = np.dtype('float32'),
+                  max_z_error: float = 0.0) -> bytes:
+    """Compress raw pixel data with LERC. Requires the ``lerc`` package.
+
+    Parameters
+    ----------
+    max_z_error : float
+        Maximum encoding error per pixel. 0 = lossless.
+    """
+    if not LERC_AVAILABLE:
+        raise ImportError(
+            "lerc is required to write LERC-compressed TIFFs. "
+            "Install it with: pip install lerc")
+    if samples == 1:
+        arr = np.frombuffer(data, dtype=dtype).reshape(height, width)
+    else:
+        arr = np.frombuffer(data, dtype=dtype).reshape(height, width, samples)
+    n_values_per_pixel = samples
+    # lerc.encode(npArr, nValuesPerPixel, bHasMask, npValidMask,
+    #             maxZErr, nBytesHint)
+    # nBytesHint=1 triggers actual encoding (0 = compute size only)
+    result = _lerc.encode(arr, n_values_per_pixel, False, None,
+                          max_z_error, 1)
+    if result[0] != 0:
+        raise RuntimeError(f"LERC encode failed with error code {result[0]}")
+    # result is (error_code, nBytesWritten, ctypes_buffer)
+    return bytes(result[2])
+
 
 # -- Dispatch helpers ---------------------------------------------------------
 
@@ -800,7 +937,9 @@ def jpeg2000_compress(data: bytes, width: int, height: int,
 COMPRESSION_DEFLATE = 8
 COMPRESSION_JPEG2000 = 34712
 COMPRESSION_ZSTD = 50000
+COMPRESSION_LZ4 = 50004
 COMPRESSION_PACKBITS = 32773
+COMPRESSION_LERC = 34887
 COMPRESSION_ADOBE_DEFLATE = 32946
 
 
@@ -839,6 +978,11 @@ def decompress(data, compression: int, expected_size: int = 0,
     elif compression == COMPRESSION_JPEG2000:
         return np.frombuffer(
             jpeg2000_decompress(data, width, height, samples), dtype=np.uint8)
+    elif compression == COMPRESSION_LZ4:
+        return np.frombuffer(lz4_decompress(data), dtype=np.uint8)
+    elif compression == COMPRESSION_LERC:
+        return np.frombuffer(
+            lerc_decompress(data, width, height, samples), dtype=np.uint8)
     else:
         raise ValueError(f"Unsupported compression type: {compression}")
 
@@ -869,9 +1013,13 @@ def compress(data: bytes, compression: int, level: int = 6) -> bytes:
         return packbits_compress(data)
     elif compression == COMPRESSION_ZSTD:
         return zstd_compress(data, level)
+    elif compression == COMPRESSION_LZ4:
+        return lz4_compress(data, level)
     elif compression == COMPRESSION_JPEG:
         raise ValueError("Use jpeg_compress() directly with width/height/samples")
     elif compression == COMPRESSION_JPEG2000:
         raise ValueError("Use jpeg2000_compress() directly with width/height/samples/dtype")
+    elif compression == COMPRESSION_LERC:
+        raise ValueError("Use lerc_compress() directly with width/height/samples/dtype")
     else:
         raise ValueError(f"Unsupported compression type: {compression}")

xrspatial/geotiff/_gpu_decode.py

@@ -1534,6 +1534,21 @@ def gpu_decode_tiles(
             decomp_offsets = np.arange(n_tiles, dtype=np.int64) * tile_bytes
             d_decomp_offsets = cupy.asarray(decomp_offsets)
 
+    elif compression == 34887:  # LERC
+        from ._compression import lerc_decompress
+        raw_host = np.empty(n_tiles * tile_bytes, dtype=np.uint8)
+        for i, tile in enumerate(compressed_tiles):
+            start = i * tile_bytes
+            chunk = np.frombuffer(
+                lerc_decompress(tile, tile_width, tile_height, samples),
+                dtype=np.uint8)
+            raw_host[start:start + min(len(chunk), tile_bytes)] = \
+                chunk[:tile_bytes] if len(chunk) >= tile_bytes else \
+                np.pad(chunk, (0, tile_bytes - len(chunk)))
+        d_decomp = cupy.asarray(raw_host)
+        decomp_offsets = np.arange(n_tiles, dtype=np.int64) * tile_bytes
+        d_decomp_offsets = cupy.asarray(decomp_offsets)
+
     elif compression == 1:  # Uncompressed
         raw_host = np.empty(n_tiles * tile_bytes, dtype=np.uint8)
         for i, tile in enumerate(compressed_tiles):
@@ -2273,6 +2288,19 @@ def gpu_compress_tiles(d_image, tile_width, tile_height,
                 samples=samples, dtype=dtype))
         return result
 
+    # LERC: CPU only, no GPU library
+    if compression == 34887:
+        from ._compression import lerc_compress
+        cpu_buf = d_tile_buf.get()
+        result = []
+        for i in range(n_tiles):
+            start = i * tile_bytes
+            tile_data = bytes(cpu_buf[start:start + tile_bytes])
+            result.append(lerc_compress(
+                tile_data, tile_width, tile_height,
+                samples=samples, dtype=dtype))
+        return result
+
     # Try nvCOMP batch compress
     result = _nvcomp_batch_compress(d_tiles, None, tile_bytes, compression, n_tiles)
 

xrspatial/geotiff/_writer.py

@@ -10,6 +10,8 @@
     COMPRESSION_DEFLATE,
     COMPRESSION_JPEG,
     COMPRESSION_JPEG2000,
+    COMPRESSION_LERC,
+    COMPRESSION_LZ4,
     COMPRESSION_LZW,
     COMPRESSION_NONE,
     COMPRESSION_PACKBITS,
@@ -71,8 +73,10 @@ def _compression_tag(compression_name: str) -> int:
         'jpeg': COMPRESSION_JPEG,
         'packbits': COMPRESSION_PACKBITS,
         'zstd': COMPRESSION_ZSTD,
+        'lz4': COMPRESSION_LZ4,
         'jpeg2000': COMPRESSION_JPEG2000,
         'j2k': COMPRESSION_JPEG2000,
+        'lerc': COMPRESSION_LERC,
     }
     name = compression_name.lower()
     if name not in _map:
@@ -332,6 +336,10 @@ def _write_stripped(data: np.ndarray, compression: int, predictor: bool,
                 from ._compression import jpeg2000_compress
                 compressed = jpeg2000_compress(
                     strip_data, width, strip_rows, samples=samples, dtype=dtype)
+            elif compression == COMPRESSION_LERC:
+                from ._compression import lerc_compress
+                compressed = lerc_compress(
+                    strip_data, width, strip_rows, samples=samples, dtype=dtype)
             else:
                 compressed = compress(strip_data, compression)
 
@@ -387,6 +395,10 @@ def _prepare_tile(data, tr, tc, th, tw, height, width, samples, dtype,
         from ._compression import jpeg2000_compress
         return jpeg2000_compress(
             tile_data, tw, th, samples=samples, dtype=dtype)
+    if compression == COMPRESSION_LERC:
+        from ._compression import lerc_compress
+        return lerc_compress(
+            tile_data, tw, th, samples=samples, dtype=dtype)
     return compress(tile_data, compression)