Tests for some specific save/load usecases.

Author: pp-mo

lib/iris/fileformats/netcdf/saver.py

@@ -1833,8 +1833,8 @@ def _create_generic_cf_array_var(
         if cube is not None and data is not None and cube.shape != data.shape:
             compression_kwargs = {}
 
-        if not is_dataless and np.issubdtype(data.dtype, np.str_):
-            # Deal with string-type variables.
+        if not is_dataless and data.dtype.kind == "U":
+            # Deal with unicode-string-type variables.
             # Typically CF label variables, but also possibly ancil-vars ?
 
             # NOTE: all we are doing here is to calculate the byte dimension length,
@@ -1843,37 +1843,26 @@ def _create_generic_cf_array_var(
             # being a _bytecoding_datasets.EncodedVariable.
             string_dimension_depth = data.dtype.itemsize
 
-            if data.dtype.kind == "U":
-                # String content (U) instead of bytes (S).
-                # For numpy strings, itemsize is **always** a multiple of 4
-                if string_dimension_depth % 4 != 0:
-                    msg = (
-                        "Unexpected numpy string 'itemsize' for element "
-                        f"{cube_or_mesh.name()}: "
-                        f"'dtype.itemsize = {string_dimension_depth}, expected "
-                        "a multiple of four (always)."
-                    )
-                    raise ValueError(msg)
-                nchars = string_dimension_depth // 4
-
-                encoding_attr = element.attributes.get("_Encoding", "ascii")
-                # Look this up + return a supported encoding name
-                # NB implements defaults and raises a warning if given not recognised.
-                encoding = bytecoding_datasets._identify_encoding(
-                    encoding=encoding_attr, var_name=cf_name, writing=True
+            # String content (U) instead of bytes (S).
+            # For numpy strings, itemsize is **always** a multiple of 4
+            if string_dimension_depth % 4 != 0:
+                msg = (
+                    "Unexpected numpy string 'itemsize' for element "
+                    f"{cube_or_mesh.name()}: "
+                    f"'dtype.itemsize = {string_dimension_depth}, expected "
+                    "a multiple of four (always)."
                 )
-                width_fns = bytecoding_datasets._ENCODING_WIDTH_TRANSLATIONS[encoding]
-                string_dimension_depth = width_fns.nchars_2_nbytes(nchars)
-            else:
-                if data.dtype.kind != "S" or data.dtype.itemsize != 1:
-                    # Some type of data we don't "understand".
-                    # NB this includes "Sxx" types other than "S1" :  It seems that
-                    # netCDF4 saves Sxx as variable-length strings.  But we don't support that type in Iris.
-                    msg = (
-                        f"Variable {cf_name!r} has unexpected string/character dtype, "
-                        f"{data.dtype} -- should be either 'S' or 'U' type."
-                    )
-                    raise ValueError(msg)
+                raise ValueError(msg)
+            nchars = string_dimension_depth // 4
+
+            encoding_attr = element.attributes.get("_Encoding", "ascii")
+            # Look this up + return a supported encoding name
+            # NB implements defaults and raises a warning if given not recognised.
+            encoding = bytecoding_datasets._identify_encoding(
+                encoding=encoding_attr, var_name=cf_name, writing=True
+            )
+            width_fns = bytecoding_datasets._ENCODING_WIDTH_TRANSLATIONS[encoding]
+            string_dimension_depth = width_fns.nchars_2_nbytes(nchars)
 
             string_dimension_name = "string%d" % string_dimension_depth
 
@@ -1893,12 +1882,34 @@ def _create_generic_cf_array_var(
             # Create the label coordinate variable.
             cf_var = self._dataset.createVariable(cf_name, "|S1", element_dims)
         else:
-            # A normal (numeric) variable.
+            # A non-string variable.
             # ensure a valid datatype for the file format.
             if is_dataless:
                 dtype = self._DATALESS_DTYPE
                 fill_value = self._DATALESS_FILLVALUE
             else:
+                # Normal non-string data.
+                # NOTE: this includes byte-arrays (S1 only) : however these must
+                # use an actual cube dimension for the 'string dimension', which
+                # seriously limits the utility of DECODE_TO_STRINGS_ON_READ.
+                # TODO: also support netCDF variable-length strings ("string" type).
+                #  Currently hit a **write error here**, being numpy object dtype ("O").
+                if (
+                    data.dtype.kind not in "iufSU"
+                    or data.dtype.kind == "S"
+                    and data.dtype.itemsize != 1
+                ):
+                    # This is a type of data we don't "understand".
+                    # NB this includes "Sxx" types other than "S1" :  It seems that
+                    # netCDF4 saves Sxx as variable-length strings.
+                    # But we don't support that type in Iris.
+                    msg = (
+                        f"Variable {cf_name!r} has unexpected dtype, {data.dtype!r}."
+                        f"Data content arrays must be numeric, or contain "
+                        "single-bytes (dtype 'S1'), or unicode strings (dtype 'U<n>')."
+                    )
+                    raise ValueError(msg)
+
                 element_type = type(element).__name__
                 data = self._ensure_valid_dtype(data, element_type, element)
                 if not packing_controls:

lib/iris/tests/integration/netcdf/test_stringdata.py

@@ -109,7 +109,10 @@ class SamplefileDetails:
 def make_testfile(
     testfile_path: Path,
     encoding_str: str,
-    coords_on_separate_dim: bool,
+    coords_on_separate_dim: bool = False,
+    # If set, determines the "_Encoding" attrs content, including None --> no attr.
+    # Otherwise, they  follow 'encoding_str', including NO_ENCODING_STR --> no attr.
+    encoding_attr: str | None = "<as_encoding_str>",
 ) -> SamplefileDetails:
     """Create a test netcdf file.
 
@@ -120,6 +123,9 @@ def make_testfile(
     else:
         encoding = encoding_str
 
+    if encoding_attr == "<as_encoding_str>":
+        encoding_attr = encoding
+
     data_is_ascii = encoding in (None, "ascii")
 
     numeric_values = np.arange(3.0)
@@ -156,8 +162,8 @@ def make_testfile(
         )
         v_co[:] = coordvar_bytearray
 
-        if encoding is not None:
-            v_co._Encoding = encoding
+        if encoding_attr is not None:
+            v_co._Encoding = encoding_attr
 
         v_numeric = ds.createVariable(
             "v_numeric",
@@ -176,8 +182,8 @@ def make_testfile(
         )
         v_datavar[:] = datavar_bytearray
 
-        if encoding is not None:
-            v_datavar._Encoding = encoding
+        if encoding_attr is not None:
+            v_datavar._Encoding = encoding_attr
 
         v_datavar.coordinates = "v_co v_numeric"
     finally:
@@ -600,3 +606,89 @@ def test_save_single_unicode__okay(self, tmp_path):
         result = iris.load_cube(filepath)
         result.attributes.pop("Conventions", None)
         assert result == scalar_char_cube
+
+
+class TestReadParticularCases:
+    @pytest.mark.parametrize("data_encoding", ["utf8", "utf16", "utf32"])
+    def test_read_no_encoding(self, tmp_path, data_encoding):
+        # Check that we can read UTF-8 encoded data, even with no _Encoding attribute.
+        # This is a common case in the wild, and now accepted by CF as a default.
+        # However, other encodings will FAIL to decode.
+        filepath = tmp_path / "utf8_no_encoding.nc"
+        testdata = make_testfile(
+            testfile_path=filepath,
+            encoding_str=data_encoding,
+            encoding_attr=None,
+        )
+        cube = iris.load_cube(filepath)
+        assert "_Encoding" not in cube.attributes
+
+        if data_encoding == "utf8":
+            assert np.all(cube.data == testdata.datavar_data)
+        else:
+            msg = "Character data .* could not be decoded with the 'utf-8' encoding"
+            with pytest.raises(ValueError, match=msg):
+                cube.data
+
+    def test_read_wrong_encoding__fail(self, tmp_path):
+        filepath = tmp_path / "missing_encoding.nc"
+        testdata = make_testfile(
+            testfile_path=filepath,
+            encoding_str="utf-16",
+            encoding_attr="utf-8",
+        )
+        cube = iris.load_cube(filepath)
+        # NOTE: error only occurs when you attempt to fetch + translate the content.
+        msg = "Character data .* could not be decoded with the 'utf-8' encoding."
+        with pytest.raises(ValueError, match=msg):
+            data = cube.data
+
+
+class TestWriteParticularCases:
+    def test_write_unicode_no_encoding__fail(self, tmp_path):
+        cube = Cube(np.array("éclair"))
+        filepath = tmp_path / "write_unicode_no_encoding.nc"
+        msg = (
+            "String data written to netcdf character variable 'unknown' "
+            "could not be represented in encoding 'ascii'"
+        )
+        with pytest.raises(ValueError, match=msg):
+            iris.save(cube, filepath)
+
+    def test_write_encoded_overlength__fail(self, tmp_path):
+        cube = Cube(np.array("éclair"), attributes={"_Encoding": "utf8"})
+        filepath = tmp_path / "write_encoded_overlength.nc"
+        msg = (
+            "String 'éclair' written into netcdf variable 'unknown' "
+            "with encoding 'utf-8' is 7 bytes long, which exceeds the "
+            "string dimension length, 6. "
+            r"This can be fixed by converting the data to a \"wider\" string dtype, "
+            r"e.g. cube.data = cube.data.astype\(\"U7\"\)"
+        )
+        with pytest.raises(iris.exceptions.TranslationError, match=msg):
+            iris.save(cube, filepath)
+
+    def test_write_multibytes__fail(self, tmp_path):
+        encoded_bytes = "éclair".encode("utf8")
+        byte_array = np.array(encoded_bytes)
+        cube = Cube(byte_array, attributes={"_Encoding": "utf8"})
+        filepath = tmp_path / "write_multibyte_Sxx.nc"
+        msg = (
+            r"Variable 'unknown' has unexpected dtype, dtype\('S7'\)."
+            "Data content arrays must be numeric, or contain single-bytes "
+            r"\(dtype 'S1'\), or unicode strings \(dtype 'U<n>'\)."
+        )
+        with pytest.raises(ValueError, match=msg):
+            iris.save(cube, filepath)
+
+    def test_write_stringobjects__fail(self, tmp_path):
+        string_array = np.array(["one", "four"], dtype="O")
+        cube = Cube(string_array)
+        filepath = tmp_path / "write_stringobjects.nc"
+        msg = (
+            r"Variable 'unknown' has unexpected dtype, dtype\('O'\)."
+            "Data content arrays must be numeric, or contain single-bytes "
+            r"\(dtype 'S1'\), or unicode strings \(dtype 'U<n>'\)."
+        )
+        with pytest.raises(ValueError, match=msg):
+            iris.save(cube, filepath)