fix: handle table data that contain NaN values

Author: paulmueller

CHANGELOG

@@ -1,3 +1,5 @@
+1.0.9
+ - fix: handle table data that contain NaN values
 1.0.8
  - fix: if mimetype not available, check suffix for detecting DC data
  - tests: own the docker image

ckanext/dc_serve/serve.py

@@ -13,6 +13,8 @@
     DC_MIME_TYPES, s3cc, is_resource_private,
 )
 
+import numpy as np
+
 
 logger = logging.getLogger(__name__)
 
@@ -43,8 +45,18 @@ def get_dc_tables(ds, from_basins: bool = False) -> dict:
                 tables.update(get_dc_tables(bn.ds))
     else:
         for tab in ds.tables:
+            # A table is a 2D array
+            tab_array = ds.tables[tab][:]
+            tab_list = tab_array.tolist()
+            if np.any(np.isnan(tab_array)):
+                # NaN is not part of the JSON specification, so we convert it
+                # to None before sending the response.
+                tab_list = [[v if not np.isnan(v) else None for v in column]
+                            for column in tab_list]
+
             tables[tab] = (ds.tables[tab].keys(),
-                           ds.tables[tab][:].tolist())
+                           tab_list
+                           )
 
     return tables
 

ckanext/dc_serve/tests/test_serve.py

@@ -8,8 +8,9 @@
 
 import ckan.tests.factories as factories
 import dclab
-from dclab.rtdc_dataset import fmt_http
+from dclab.rtdc_dataset import fmt_dcor, fmt_http
 import h5py
+import numpy as np
 
 import pytest
 
@@ -606,6 +607,72 @@ def test_api_dcserv_tables(enqueue_job_mock, app):
     assert "brightness" in names
 
 
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas dc_serve')
+@pytest.mark.usefixtures('with_plugins', 'clean_db')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_api_dcserv_tables_with_nan(enqueue_job_mock, app, tmp_path):
+    """
+    Tables may contain NaN-data. These should be converted into None
+    before they are encoded with JSON and sent as response.
+    """
+    user = factories.UserWithToken()
+    owner_org = factories.Organization(users=[{
+        'name': user['id'],
+        'capacity': 'admin'
+    }])
+    # Note: `call_action` bypasses authorization!
+    create_context = {'ignore_auth': False,
+                      'user': user['name'], 'api_version': 3}
+
+    # generate a resource with nan-valued tables
+    path_mod = tmp_path / 'blood_nan.rtdc'
+    shutil.copy2(data_path / "cytoshot_blood.rtdc", path_mod)
+
+    # generate a table
+    table_data = {
+        "fox": np.arange(10),
+        "peter": np.linspace(-1, 1, 10),
+        "deril": np.zeros(10),
+    }
+    # this is the important bit for this test
+    table_data["peter"][2] = np.nan
+
+    with dclab.RTDCWriter(path_mod) as hw:
+        hw.store_table("nan-array", table_data)
+
+    # create a dataset
+    ds_dict, res_dict = make_dataset_via_s3(
+        create_context=create_context,
+        owner_org=owner_org,
+        resource_path=path_mod,
+        activate=True)
+
+    resp = app.get(
+        "/api/3/action/dcserv",
+        params={"id": res_dict["id"],
+                "query": "tables",
+                },
+        headers={"Authorization": user["token"]},
+        status=200
+    )
+    jres = json.loads(resp.body)
+    assert jres["success"]
+    assert "nan-array" in jres["result"]
+    names, data = jres["result"]["nan-array"]
+    assert "peter" in names
+    assert data[1][2] is None
+
+    # now open the same dataset with dclab
+    host = f"http://{app.config['CKAN_HOST']}:{app.config['CKAN_PORT']}"
+    ds = fmt_dcor.RTDC_DCOR(
+        url=f"{host}/api/3/action/dcserv?id={res_dict['id']}&query=tables",
+        api_key=user["token"])
+    assert "nan-array" in ds.tables
+    assert np.isnan(ds.tables["nan-array"]["peter"][2])
+    assert not np.isnan(ds.tables["nan-array"]["peter"][1])
+
+
 @pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas dc_serve')
 @pytest.mark.usefixtures('with_plugins', 'clean_db')
 @mock.patch('ckan.plugins.toolkit.enqueue_job',