Merge pull request #1561 from lesserwhirls/gh-1556

Extend limited reader in H5headerNew to handle filtered data

Author: lesserwhirls

cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
@@ -25,6 +25,7 @@
 import ucar.ma2.ArrayStructure;
 import ucar.ma2.ArrayStructureBB;
 import ucar.ma2.DataType;
+import ucar.ma2.Index;
 import ucar.ma2.IndexIterator;
 import ucar.ma2.InvalidRangeException;
 import ucar.ma2.Section;
@@ -59,6 +60,7 @@
 import ucar.nc2.write.NetcdfFileFormat;
 import ucar.nc2.iosp.IospHelper;
 import ucar.nc2.iosp.Layout;
+import ucar.nc2.iosp.LayoutBB;
 import ucar.nc2.iosp.LayoutRegular;
 import ucar.nc2.iosp.hdf5.DataBTree;
 import ucar.nc2.iosp.hdf5.H5headerIF;
@@ -1957,6 +1959,29 @@ Object getFillValueNonDefault() {
     Array readArray() throws IOException {
       int[] shape = mds.dimLength;
       DataType dataType = typeInfo.dataType;
+
+      if (useFillValue) {
+        Object pa = IospHelper.makePrimitiveArray((int) Index.computeSize(shape), dataType, getFillValue());
+        if (dataType == DataType.CHAR)
+          pa = IospHelper.convertByteToChar((byte[]) pa);
+        return Array.factory(dataType, shape, pa);
+      }
+
+      // filtered, must read and decode by chunk
+      if (mfp != null) {
+        ByteOrder bo = (typeInfo.endian == 0) ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN;
+        LayoutBB layoutBB;
+        try {
+          layoutBB = new H5tiledLayoutBB(this, shape, dataType.getSize(), new Section(shape), getRandomAccessFile(),
+              mfp.getFilters(), bo);
+        } catch (InvalidRangeException e) {
+          // should not happen because we passed in the full shape
+          throw new IllegalStateException();
+        }
+        Object data = IospHelper.readDataFill(layoutBB, dataType, getFillValue());
+        return Array.factory(dataType, shape, data);
+      }
+
       Layout layout;
       try {
         if (isChunked) {
@@ -1977,6 +2002,15 @@ Array readArray() throws IOException {
     String readString() throws IOException {
       int[] shape = new int[] {mdt.byteSize};
       DataType dataType = typeInfo.dataType;
+
+      if (useFillValue) {
+        Object pa = IospHelper.makePrimitiveArray((int) Index.computeSize(shape), dataType, getFillValue());
+        if (dataType == DataType.CHAR)
+          pa = IospHelper.convertByteToChar((byte[]) pa);
+        Array dataArray = Array.factory(dataType, shape, pa);
+        return (dataArray instanceof ArrayChar.D1) ? ((ArrayChar) dataArray).getString() : "";
+      }
+
       Layout layout;
       try {
         if (isChunked) {

cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5tiledLayoutBB.java

@@ -1,7 +1,8 @@
 /*
- * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
+
 package ucar.nc2.internal.iosp.hdf5;
 
 import com.google.common.primitives.Ints;
@@ -71,9 +72,27 @@ public class H5tiledLayoutBB implements LayoutBB {
    */
   public H5tiledLayoutBB(Variable v2, Section wantSection, RandomAccessFile raf, H5objects.Filter[] filterProps,
       ByteOrder byteOrder) throws InvalidRangeException, IOException {
-    wantSection = Section.fill(wantSection, v2.getShape());
+    this((H5headerNew.Vinfo) v2.getSPobject(), v2.getShape(), v2.getElementSize(), wantSection, raf, filterProps,
+        byteOrder);
+  }
+
+  /**
+   * This constructor can be used when the Variable is not yet built.
+   *
+   * @param vinfo the data object
+   * @param varShape the variable's shape
+   * @param elemSize the variable's element size in bytes
+   * @param wantSection the wanted section of data, contains a List of Range objects. must be complete
+   * @param raf the RandomAccessFile
+   * @param filterProps set of filter properties from which filter object will be created
+   * @throws InvalidRangeException if section invalid for this variable
+   * @throws IOException on io error
+   */
+  H5tiledLayoutBB(H5headerNew.Vinfo vinfo, int[] varShape, int elemSize, Section wantSection, RandomAccessFile raf,
+      H5objects.Filter[] filterProps, ByteOrder byteOrder) throws InvalidRangeException, IOException {
+
+    wantSection = Section.fill(wantSection, varShape);
 
-    H5headerNew.Vinfo vinfo = (H5headerNew.Vinfo) v2.getSPobject();
     assert vinfo.isChunked;
     assert vinfo.btree != null;
 
@@ -82,7 +101,7 @@ public H5tiledLayoutBB(Variable v2, Section wantSection, RandomAccessFile raf, H
     for (int i = 0; i < filterProps.length; i++) {
       // add var info to filter props
       Map<String, Object> props = filterProps[i].getProperties();
-      props.put(Filters.Keys.ELEM_SIZE, v2.getElementSize());
+      props.put(Filters.Keys.ELEM_SIZE, elemSize);
       // try to get filter by name or id, throw if not recognized filter
       try {
         filters[i] = Filters.getFilter(props);
@@ -95,7 +114,7 @@ public H5tiledLayoutBB(Variable v2, Section wantSection, RandomAccessFile raf, H
     // we have to translate the want section into the same rank as the storageSize, in order to be able to call
     // Section.intersect(). It appears that storageSize (actually msl.chunkSize) may have an extra dimension, relative
     // to the Variable.
-    DataType dtype = v2.getDataType();
+    DataType dtype = vinfo.getNCDataType();
     if ((dtype == DataType.CHAR) && (wantSection.getRank() < vinfo.storageSize.length)) {
       this.want = Section.builder().appendRanges(wantSection.getRanges()).appendRange(1).build();
     } else {