WIP: changing saver string handling.

Author: pp-mo

lib/iris/fileformats/netcdf/saver.py

@@ -1833,7 +1833,9 @@ def _create_generic_cf_array_var(
         if cube is not None and data is not None and cube.shape != data.shape:
             compression_kwargs = {}
 
-        if not is_dataless and np.issubdtype(data.dtype, np.str_):
+        if not is_dataless and np.issubdtype(
+            data.dtype, np.str_
+        ):  # data.dtype.kind in "SU":
             # Deal with string-type variables.
             # Typically CF label variables, but also possibly ancil-vars ?
 
@@ -1843,7 +1845,8 @@ def _create_generic_cf_array_var(
             # being a _bytecoding_datasets.EncodedVariable.
             string_dimension_depth = data.dtype.itemsize
 
-            if data.dtype.kind == "U":
+            assert data.dtype.kind == "U"
+            if 1:
                 # String content (U) instead of bytes (S).
                 # For numpy strings, itemsize is **always** a multiple of 4
                 if string_dimension_depth % 4 != 0:
@@ -1864,16 +1867,6 @@ def _create_generic_cf_array_var(
                 )
                 width_fns = bytecoding_datasets._ENCODING_WIDTH_TRANSLATIONS[encoding]
                 string_dimension_depth = width_fns.nchars_2_nbytes(nchars)
-            else:
-                if data.dtype.kind != "S" or data.dtype.itemsize != 1:
-                    # Some type of data we don't "understand".
-                    # NB this includes "Sxx" types other than "S1" :  It seems that
-                    # netCDF4 saves Sxx as variable-length strings.  But we don't support that type in Iris.
-                    msg = (
-                        f"Variable {cf_name!r} has unexpected string/character dtype, "
-                        f"{data.dtype} -- should be either 'S' or 'U' type."
-                    )
-                    raise ValueError(msg)
 
             string_dimension_name = "string%d" % string_dimension_depth
 

lib/iris/tests/integration/netcdf/test_stringdata.py

@@ -109,7 +109,10 @@ class SamplefileDetails:
 def make_testfile(
     testfile_path: Path,
     encoding_str: str,
-    coords_on_separate_dim: bool,
+    coords_on_separate_dim: bool = False,
+    # If set, determines the "_Encoding" attrs content, including None --> no attr.
+    # Otherwise, they  follow 'encoding_str', including NO_ENCODING_STR --> no attr.
+    encoding_attr: str | None = "<as_encoding_str>",
 ) -> SamplefileDetails:
     """Create a test netcdf file.
 
@@ -120,6 +123,9 @@ def make_testfile(
     else:
         encoding = encoding_str
 
+    if encoding_attr == "<as_encoding_str>":
+        encoding_attr = encoding
+
     data_is_ascii = encoding in (None, "ascii")
 
     numeric_values = np.arange(3.0)
@@ -156,8 +162,8 @@ def make_testfile(
         )
         v_co[:] = coordvar_bytearray
 
-        if encoding is not None:
-            v_co._Encoding = encoding
+        if encoding_attr is not None:
+            v_co._Encoding = encoding_attr
 
         v_numeric = ds.createVariable(
             "v_numeric",
@@ -176,8 +182,8 @@ def make_testfile(
         )
         v_datavar[:] = datavar_bytearray
 
-        if encoding is not None:
-            v_datavar._Encoding = encoding
+        if encoding_attr is not None:
+            v_datavar._Encoding = encoding_attr
 
         v_datavar.coordinates = "v_co v_numeric"
     finally:
@@ -600,3 +606,82 @@ def test_save_single_unicode__okay(self, tmp_path):
         result = iris.load_cube(filepath)
         result.attributes.pop("Conventions", None)
         assert result == scalar_char_cube
+
+
+class TestReadSpecialCases:
+    @pytest.mark.parametrize("data_encoding", ["utf8", "utf16", "utf32"])
+    def test_read_no_encoding(self, tmp_path, data_encoding):
+        # Check that we can read UTF-8 encoded data, even with no _Encoding attribute.
+        # This is a common case in the wild, and now accepted by CF as a default.
+        filepath = tmp_path / "utf8_no_encoding.nc"
+        testdata = make_testfile(
+            testfile_path=filepath,
+            encoding_str=data_encoding,
+            encoding_attr=None,
+        )
+        cube = iris.load_cube(filepath)
+        assert "_Encoding" not in cube.attributes
+
+        if data_encoding == "utf8":
+            assert np.all(cube.data == testdata.datavar_data)
+        else:
+            msg = "Character data .* could not be decoded with the 'utf-8' encoding"
+            with pytest.raises(ValueError, match=msg):
+                cube.data
+
+    def test_read_wrong_encoding__fail(self, tmp_path):
+        filepath = tmp_path / "missing_encoding.nc"
+        testdata = make_testfile(
+            testfile_path=filepath,
+            encoding_str="utf-16",
+            encoding_attr="utf-8",
+        )
+
+        from pp_utils import ncdump
+
+        ncdump(filepath, opts="-s")
+
+        cube = iris.load_cube(filepath)
+        # NOTE: error only occurs when you attempt to fetch + translate the content.
+        msg = "Character data .* could not be decoded with the 'utf-8' encoding."
+        with pytest.raises(ValueError, match=msg):
+            data = cube.data
+
+
+class TestWriteSpecialCases:
+    def test_write_unicode_no_encoding__fail(self, tmp_path):
+        cube = Cube(np.array("éclair"))
+        filepath = tmp_path / "write_unicode_no_encoding.nc"
+        msg = (
+            "String data written to netcdf character variable 'unknown' "
+            "could not be represented in encoding 'ascii'"
+        )
+        with pytest.raises(ValueError, match=msg):
+            iris.save(cube, filepath)
+
+    def test_write_encoded_overlength__fail(self, tmp_path):
+        cube = Cube(np.array("éclair"), attributes={"_Encoding": "utf8"})
+        filepath = tmp_path / "write_encoded_overlength.nc"
+        msg = (
+            "String 'éclair' written into netcdf variable 'unknown' "
+            "with encoding 'utf-8' is 7 bytes long, which exceeds the "
+            "string dimension length, 6. "
+            r"This can be fixed by converting the data to a \"wider\" string dtype, "
+            r"e.g. cube.data = cube.data.astype\(\"U7\"\)"
+        )
+        with pytest.raises(iris.exceptions.TranslationError, match=msg):
+            iris.save(cube, filepath)
+
+    @pytest.skip("Under development")
+    def test_write_multibytes__fail(self, tmp_path):
+        encoded_bytes = "éclair".encode("utf8")
+        byte_array = np.array(encoded_bytes)
+        cube = Cube(byte_array, attributes={"_Encoding": "utf8"})
+        filepath = tmp_path / "write_multibyte_Sxx.nc"
+        msg = (
+            r"Variable 'unknown' has unexpected string/character type, dtype\('S7'\) "
+            r"-- the data array must contain either single-bytes \(dtype 'S1'\), "
+            r"or unicode strings \(dtype 'U<n>'\)"
+        )
+        with pytest.raises(ValueError, match=msg):
+            iris.save(cube, filepath)