Add "Text" Khiops type assignment heuristic

Thus, "Text" is assigned to a dataframe input column if:
- its type is StringDType, and
- its length is > 100 (same heuristic as Khiops core when called through
  `api.build_dictionary_from_data_table`).

Author: popescu-v

khiops/sklearn/dataset.py

@@ -236,18 +236,25 @@ def _upgrade_mapping_spec(ds_spec):
     return new_ds_spec
 
 
-def get_khiops_type(numpy_type):
+def get_khiops_type(numpy_type, max_type_size=None):
     """Translates a numpy dtype to a Khiops dictionary type
 
     Parameters
     ----------
-    numpy_type : `numpy.dtype`:
+    numpy_type : `numpy.dtype`
         Numpy type of the column
+    max_type_size : `int`, optional
+        Maximum length of the entries of the column whose type is ``numpy_type``.
 
     Returns
     -------
     str
-        Khiops type name. Either "Categorical", "Numerical" or "Timestamp"
+        Khiops type name. Either "Categorical", "Text", "Numerical" or "Timestamp".
+
+    .. note::
+        The "Text" Khiops type is inferred if the Numpy type is "string"
+        and the maximum length of the entries of that type is greater than 100.
+
     """
     lower_numpy_type = str(numpy_type).lower()
 
@@ -257,6 +264,18 @@ def get_khiops_type(numpy_type):
     # float<x>, int<x>, uint<x> types
     elif "int" in lower_numpy_type or "float" in lower_numpy_type:
         khiops_type = "Numerical"
+    elif lower_numpy_type == "string":
+        if max_type_size is not None:
+            if not isinstance(max_type_size, (int, np.int64)):
+                raise TypeError(
+                    type_error_message("max_type_size", max_type_size, int, np.int64)
+                )
+            if max_type_size > 100:
+                khiops_type = "Text"
+            else:
+                khiops_type = "Categorical"
+        else:
+            khiops_type = "Categorical"
     # bool_ and object, character, bytes_, str_, void, record and other types
     else:
         khiops_type = "Categorical"
@@ -956,10 +975,14 @@ def __init__(self, name, dataframe, key=None):
                 )
 
         # Initialize Khiops types
-        self.khiops_types = {
-            column_id: get_khiops_type(self.data_source.dtypes[column_id])
-            for column_id in self.column_ids
-        }
+        self.khiops_types = {}
+        for column_id in self.column_ids:
+            column = self.data_source[column_id]
+            column_numpy_type = column.dtype
+            column_max_size = column.astype("str").str.len().max()
+            self.khiops_types[column_id] = get_khiops_type(
+                column_numpy_type, column_max_size
+            )
 
         # Check key integrity
         self.check_key()

tests/test_dataset_class.py

@@ -100,15 +100,17 @@ def create_monotable_dataframe(self):
                 False,
                 False,
             ],
+            # Make some entries longer than 100 characters in the "Title" column
+            # to trigger the "Text" Khiops type assignment heuristic
             "Title": [
-                "Awesome",
+                (15 * "Awesome ").strip(),
                 "Very lovely",
                 "Some major design flaws",
                 "My favorite buy!",
-                "Flattering shirt",
+                (7 * "Flattering shirt ").strip(),
                 "Not for the very petite",
                 "Cagrcoal shimmer fun",
-                "Shimmer, surprisingly goes with lots",
+                (3 * "Shimmer, surprisingly goes with lots ").strip(),
                 "Flattering",
                 "Such a fun dress!",
             ],
@@ -128,6 +130,9 @@ def create_monotable_dataframe(self):
             ],
         }
         dataset = pd.DataFrame(data)
+
+        # Force StringDType on "Title" to have it of "Text" Khiops type
+        dataset["Title"] = dataset["Title"].astype("string")
         return dataset
 
     def create_monotable_data_file(self, table_path):
@@ -294,7 +299,7 @@ def get_ref_var_types(self, multitable, schema=None):
                 "Clothing ID": "Numerical",
                 "Date": "Timestamp",
                 "New": "Categorical",
-                "Title": "Categorical",
+                "Title": "Text",
                 "Recommended IND": "Numerical",
                 "Positive Feedback average": "Numerical",
                 "class": "Categorical",
@@ -489,7 +494,9 @@ def test_out_file_from_dataframe_monotable(self):
 
         # Create and load the intermediary Khiops file
         out_table_path, _ = dataset.create_table_files_for_khiops(self.output_dir)
-        out_table = pd.read_csv(out_table_path, sep="\t")
+
+        # Force StringDType on the "Title" column upon CSV reading
+        out_table = pd.read_csv(out_table_path, sep="\t", dtype={"Title": "string"})
 
         # Cast "Date" columns to datetime as we don't automatically recognize dates
         out_table["Date"] = out_table["Date"].astype("datetime64[ns]")
@@ -799,7 +806,7 @@ def create_monotable_dataset_with_newlines():
             ],
             "Age": [39],
             "Title": [
-                "Shimmer,\nsurprisingly\n\rgoes with lots",
+                (3 * "Shimmer,\nsurprisingly\n\rgoes with lots").strip(),
             ],
         }
         dataset = pd.DataFrame(data)
@@ -814,7 +821,7 @@ def test_newlines_removed_from_csv_file_for_khiops(self):
         out_table = pd.read_csv(out_table_path, sep="\t")
 
         self.assertEqual(
-            "Shimmer, surprisingly  goes with lots",
+            (3 * "Shimmer, surprisingly  goes with lots").strip(),
             out_table.Title[0],
             "Newlines should have been removed from the data",
         )