Checks for alternate encoding name.

Author: pp-mo

lib/iris/tests/integration/netcdf/test_stringdata.py

@@ -38,11 +38,12 @@ def all_lazy_auxcoords():
 N_XDIM = 3
 N_CHARS_DIM = 64
 # TODO: remove (debug)
-# PERSIST_TESTFILES: str | None = "~/chararray_testfiles"
-PERSIST_TESTFILES: str | None = None
+PERSIST_TESTFILES: str | None = "~/chararray_testfiles"
+# PERSIST_TESTFILES: str | None = None
 
 NO_ENCODING_STR = "<noencoding>"
-TEST_ENCODINGS = [NO_ENCODING_STR] + SUPPORTED_ENCODINGS
+ALIAS_UTF8_STR = "UTF8"  # an alternative acceptable form (should be written as-is)
+TEST_ENCODINGS = [NO_ENCODING_STR, ALIAS_UTF8_STR] + SUPPORTED_ENCODINGS
 
 
 #
@@ -223,7 +224,7 @@ def readtest_path(
         else:
             filetag = encoding
         dimtag = "diffdims" if use_separate_dims else "samedims"
-        tempfile_path = tmp_path / f"sample_read_{filetag}_{dimtag}.nc"
+        tempfile_path = tmp_path / f"sample_stringdata_read_{filetag}_{dimtag}.nc"
         return tempfile_path
 
     @pytest.fixture
@@ -408,7 +409,7 @@ def write_bytes(self, request):
         return request.param == "dataAsBytes"
 
     @pytest.fixture
-    def writetest_path(self, encoding, write_bytes, tmp_path):
+    def writetest_path(self, encoding, write_bytes, lazy_data, tmp_path):
         """Create a suitable test cube, with either string or byte content."""
         if PERSIST_TESTFILES:
             tmp_path = Path(PERSIST_TESTFILES).expanduser()
@@ -417,7 +418,10 @@ def writetest_path(self, encoding, write_bytes, tmp_path):
         else:
             filetag = encoding
         datatag = "writebytes" if write_bytes else "writestrings"
-        tempfile_path = tmp_path / f"sample_write_{filetag}_{datatag}.nc"
+        lazytag = "alllazy" if lazy_data else "smallreal"
+        tempfile_path = (
+            tmp_path / f"sample_stringdata_write_{filetag}_{datatag}_{lazytag}.nc"
+        )
         return tempfile_path
 
     @pytest.fixture
@@ -440,11 +444,6 @@ def writetest_data(self, writetest_path, encoding, write_bytes, lazy_data):
     def test_valid_encodings(self, encoding, writetest_data, write_bytes):
         cube_info = writetest_data
         cube, path = cube_info.cube, cube_info.save_path
-        # TODO: not testing the "byte read/write" yet
-        # Make a quick check for cube equality : but the presentation depends on the read mode
-        # with DECODE_TO_STRINGS_ON_READ.context(not write_bytes):
-        # read_cube = iris.load_cube(path)
-        # assert read_cube == cube
 
         # N.B. file content should not depend on whether bytes or strings were written
         vararray, coordarray = cube_info.datavar_data, cube_info.stringcoord_data

lib/iris/tests/unit/fileformats/netcdf/test_bytecoding_datasets.py

@@ -28,7 +28,10 @@
 from iris.tests.stock.netcdf import ncgen_from_cdl
 from iris.warnings import IrisCfLoadWarning, IrisCfSaveWarning
 
-encoding_options = [None] + SUPPORTED_ENCODINGS
+# Note: for test options, include "no encoding" and an alias name
+ENCODING_NONE = None
+ENCODING_UTF8_ALIAS = "UTF8"
+encoding_options = [ENCODING_NONE, ENCODING_UTF8_ALIAS] + SUPPORTED_ENCODINGS
 
 samples_3_ascii = np.array(
     ["one", "", "seven"],  # N.B. include empty!
@@ -45,9 +48,17 @@ def encoding(request):
     return request.param
 
 
+# TODO: remove (debug)
+PERSIST_TESTFILES: str | None = "~/chararray_testfiles"
+# PERSIST_TESTFILES: str | None = None
+
+
 @pytest.fixture(scope="module")
 def tempdir(tmp_path_factory):
-    path = tmp_path_factory.mktemp("netcdf")
+    if PERSIST_TESTFILES:
+        path = Path(PERSIST_TESTFILES).expanduser()
+    else:
+        path = tmp_path_factory.mktemp("netcdf")
     return path
 
 
@@ -134,7 +145,7 @@ class TestWriteStrings:
 
     def test_encodings(self, encoding, tempdir):
         # Create a dataset with the variable
-        path = tempdir / f"test_writestrings_encoding_{encoding!s}.nc"
+        path = tempdir / f"test_bytecoded_writestrings_encoding_{encoding!s}.nc"
 
         if encoding in [None, "ascii"]:
             writedata = samples_3_ascii
@@ -164,7 +175,7 @@ def test_encodings(self, encoding, tempdir):
 
     def test_scalar(self, tempdir):
         # Like 'test_write_strings', but the variable has *only* the string dimension.
-        path = tempdir / "test_writestrings_scalar.nc"
+        path = tempdir / "test_bytecoded_writestrings_scalar.nc"
 
         strlen = 5
         ds_encoded = make_encoded_dataset(path, strlen=strlen)
@@ -180,7 +191,7 @@ def test_scalar(self, tempdir):
 
     def test_multidim(self, tempdir):
         # Like 'test_write_strings', but the variable has additional dimensions.
-        path = tempdir / "test_writestrings_multidim.nc"
+        path = tempdir / "test_bytecoded_writestrings_multidim.nc"
 
         strlen = 5
         ds_encoded = make_encoded_dataset(path, strlen=strlen)
@@ -209,7 +220,7 @@ def test_multidim(self, tempdir):
 
     @pytest.mark.parametrize("encoding", [None, "ascii"])
     def test_write_encoding_failure(self, tempdir, encoding):
-        path = tempdir / f"test_writestrings_encoding_{encoding}_fail.nc"
+        path = tempdir / f"test_bytecoded_writestrings_encoding_{encoding}_fail.nc"
         ds = make_encoded_dataset(path, strlen=5, encoding=encoding)
         v = ds.variables["vxs"]
         encoding_name = encoding
@@ -223,7 +234,7 @@ def test_write_encoding_failure(self, tempdir, encoding):
             v[:] = samples_3_nonascii
 
     def test_write_badencoding_ignore(self, tempdir):
-        path = tempdir / "test_writestrings_badencoding_ignore.nc"
+        path = tempdir / "test_bytecoded_writestrings_badencoding_ignore.nc"
         ds = make_encoded_dataset(path, strlen=5, encoding="unknown")
         v = ds.variables["vxs"]
         msg = (
@@ -235,7 +246,7 @@ def test_write_badencoding_ignore(self, tempdir):
 
     def test_overlength(self, tempdir):
         # Check expected behaviour with over-length data
-        path = tempdir / "test_writestrings_overlength.nc"
+        path = tempdir / "test_bytecoded_writestrings_overlength.nc"
         strlen = 6
         ds = make_encoded_dataset(path, strlen=strlen, encoding="utf8")
         v = ds.variables["vxs"]
@@ -248,7 +259,7 @@ def test_overlength(self, tempdir):
 
     def test_overlength_splitcoding(self, tempdir):
         # Check expected behaviour when non-ascii multibyte coding gets truncated
-        path = tempdir / "test_writestrings_overlength_splitcoding.nc"
+        path = tempdir / "test_bytecoded_writestrings_overlength_splitcoding.nc"
         strlen = 5
         ds = make_encoded_dataset(path, strlen=strlen, encoding="utf-8")
         v = ds.variables["vxs"]
@@ -291,7 +302,7 @@ def test_write_chars(self, tempdir, write_form):
         strlen = strings_maxbytes(write_strings, encoding)
         write_bytes = make_bytearray(write_strings, strlen, encoding=encoding)
         # NOTE: 'flexi' form util decides the width needs to be 7 !!
-        path = tempdir / f"test_writechars_{write_form}.nc"
+        path = tempdir / f"test_bytecoded_writechars_{write_form}.nc"
         ds = make_encoded_dataset(path, encoding=encoding, strlen=strlen)
         v = ds.variables["vxs"]
 
@@ -331,7 +342,7 @@ def undecoded_testvar(self, ds_encoded, varname: str):
 
     def test_encodings(self, encoding, tempdir, readmode):
         # Create a dataset with the variable
-        path = tempdir / f"test_read_encodings_{encoding!s}_{readmode}.nc"
+        path = tempdir / f"test_bytecoded_read_encodings_{encoding!s}_{readmode}.nc"
 
         if encoding in [None, "ascii"]:
             write_strings = samples_3_ascii
@@ -352,10 +363,10 @@ def test_encodings(self, encoding, tempdir, readmode):
             # Test "normal" read --> string array
             result = v[:]
             expected = write_strings
-            if encoding in ("utf-8", "utf-16"):
+            if encoding in ("utf-8", ENCODING_UTF8_ALIAS, "utf-16"):
                 # In these cases, with the given non-ascii sample data, the
                 #  "default minimum string length" is overestimated.
-                if encoding == "utf-8":
+                if encoding in ["utf-8", ENCODING_UTF8_ALIAS]:
                     assert strlen == 7
                     assert result.dtype == "U7"
                     # correct the result dtype to pass the write_strings comparison below
@@ -378,7 +389,7 @@ def test_encodings(self, encoding, tempdir, readmode):
 
     def test_scalar(self, tempdir, readmode):
         # Like 'test_write_strings', but the variable has *only* the string dimension.
-        path = tempdir / f"test_read_scalar_{readmode}.nc"
+        path = tempdir / f"test_bytecoded_read_scalar_{readmode}.nc"
 
         strlen = 5
         ds_encoded = make_encoded_dataset(path, strlen=strlen)
@@ -404,7 +415,7 @@ def test_scalar(self, tempdir, readmode):
 
     def test_multidim(self, tempdir, readmode):
         # Like 'test_write_strings', but the variable has additional dimensions.
-        path = tempdir / f"test_read_multidim_{readmode}.nc"
+        path = tempdir / f"test_bytecoded_read_multidim_{readmode}.nc"
 
         strlen = 5
         ds_encoded = make_encoded_dataset(path, strlen=strlen)
@@ -440,7 +451,7 @@ def test_multidim(self, tempdir, readmode):
         check_array_matching(result, expected)
 
     def test_read_encoding_failure(self, tempdir, readmode):
-        path = tempdir / f"test_read_encoding_failure_{readmode}.nc"
+        path = tempdir / f"test_bytecoded_read_encoding_failure_{readmode}.nc"
         strlen = 10
         ds_encoded = make_encoded_dataset(path, strlen=strlen, encoding="ascii")
         v = ds_encoded.variables["vxs"]
@@ -463,7 +474,7 @@ def test_read_encoding_failure(self, tempdir, readmode):
             assert np.all(result == test_utf8_bytes)
 
     def test_read_badencoding_ignore(self, tempdir):
-        path = tempdir / f"test_read_badencoding_ignore.nc"
+        path = tempdir / f"test_bytecoded_read_badencoding_ignore.nc"
         strlen = 10
         ds = make_encoded_dataset(path, strlen=strlen, encoding="unknown")
         v = ds.variables["vxs"]