Add a way to add a variable to a dictionary using a complete specification

Author: Thierry RAMORASOAVINA

CHANGELOG.md

@@ -13,6 +13,7 @@
   comments, and dictionary and variable block internal comments.
 - (`core`) Dictionary `Rule` class and supporting API for adding and getting
   rules to / from variables and variable blocks.
+- (`core`) New way to add a variable to a dictionary using a complete specification.
 - (`sklearn`) `Text` Khiops type support at the estimator level.
 
 ### Fixed

doc/samples/samples.rst

@@ -79,40 +79,28 @@ Samples
         {"name": "ts", "type": "TimestampTZ"},
     ]
     for var_spec in simple_variables:
-        var = kh.Variable()
-        var.name = var_spec["name"]
-        var.type = var_spec["type"]
-        root_dictionary.add_variable(var)
+        root_dictionary.add_variable_from_spec(name=var_spec["name"], type=var_spec["type"])
 
     # Create a second dictionary
     second_dictionary = kh.Dictionary(
         json_data={"name": "Service", "key": ["Id", "id_product"]}
     )
-    second_dictionary.add_variable(
-        kh.Variable(json_data={"name": "Id", "type": "Categorical"})
-    )
-    second_dictionary.add_variable(
-        kh.Variable(json_data={"name": "id_product", "type": "Categorical"})
-    )
+    second_dictionary.add_variable_from_spec(name="Id", type="Categorical")
+    second_dictionary.add_variable_from_spec(name="id_product", type="Categorical")
+
     # Create a third dictionary
     third_dictionary = kh.Dictionary(json_data={"name": "Address", "key": ["Id"]})
-    third_dictionary.add_variable(
-        kh.Variable(json_data={"name": "StreetNumber", "type": "Numerical"})
-    )
-    third_dictionary.add_variable(
-        kh.Variable(json_data={"name": "StreetName", "type": "Categorical"})
-    )
-    third_dictionary.add_variable(
-        kh.Variable(json_data={"name": "id_city", "type": "Categorical"})
-    )
+    third_dictionary.add_variable_from_spec(name="StreetNumber", type="Numerical")
+    third_dictionary.add_variable_from_spec(name="StreetName", type="Categorical")
+    third_dictionary.add_variable_from_spec(name="id_city", type="Categorical")
 
     # Add the variables used in a multi-table context in the first dictionary.
     # They link the root dictionary to the additional ones
-    root_dictionary.add_variable(
-        kh.Variable(json_data={"name": "Services", "type": "Table(Service)"})
+    root_dictionary.add_variable_from_spec(
+        name="Services", type="Table", object_type="Service"
     )
-    root_dictionary.add_variable(
-        kh.Variable(json_data={"name": "Address", "type": "Entity(Address)"})
+    root_dictionary.add_variable_from_spec(
+        name="Address", type="Entity", object_type="Address"
     )
 
     # Create a DictionaryDomain (set of dictionaries)
@@ -651,25 +639,19 @@ Samples
 
     # Add a random fold index variable to the learning dictionary
     fold_number = 5
-    fold_index_variable = kh.Variable()
-    fold_index_variable.name = "FoldIndex"
-    fold_index_variable.type = "Numerical"
-    fold_index_variable.used = False
-    dictionary.add_variable(fold_index_variable)
+    dictionary.add_variable_from_spec(name="FoldIndex", type="Numerical", used=False)
 
     # Create fold indexing rule and set it on `fold_index_variable`
-    dictionary.get_variable(fold_index_variable.name).set_rule(
+    fold_index_variable = dictionary.get_variable("FoldIndex")
+    fold_index_variable.set_rule(
         kh.Rule("Ceil", kh.Rule("Product", fold_number, kh.Rule("Random()"))),
     )
 
     # Add variables that indicate if the instance is in the train dataset:
     for fold_index in range(1, fold_number + 1):
-        is_in_train_dataset_variable = kh.Variable()
-        is_in_train_dataset_variable.name = "IsInTrainDataset" + str(fold_index)
-        is_in_train_dataset_variable.type = "Numerical"
-        is_in_train_dataset_variable.used = False
-        dictionary.add_variable(is_in_train_dataset_variable)
-        dictionary.get_variable(is_in_train_dataset_variable.name).set_rule(
+        name = "IsInTrainDataset" + str(fold_index)
+        dictionary.add_variable_from_spec(name=name, type="Numerical", used=False)
+        dictionary.get_variable(name).set_rule(
             kh.Rule("NEQ", fold_index_variable, fold_index),
         )
 

khiops/core/dictionary.py

@@ -23,6 +23,8 @@
 from khiops.core.exceptions import KhiopsJSONError
 from khiops.core.internals.common import (
     deprecation_message,
+    is_dict_like,
+    is_list_like,
     is_string_like,
     type_error_message,
 )
@@ -43,25 +45,17 @@ def _format_name(name):
 
     Otherwise, it returns the name between backquoted (backquotes within are doubled)
     """
-    # Check that the type of name is string or bytes
-    if not is_string_like(name):
-        raise TypeError(type_error_message("name", name, "string-like"))
+    is_valid_identifier = _check_name(name)
 
-    # Check if the name is an identifier
-    # Python isalnum is not used because of utf-8 encoding (accentuated chars
-    # are considered alphanumeric)
     # Return original name if is an identifier, otherwise between backquotes
-    identifier_pattern = r"^[a-zA-Z_][a-zA-Z0-9_]*"
-    str_identifier_regex = re.compile(identifier_pattern)
-    bytes_identifier_regex = re.compile(bytes(identifier_pattern, encoding="ascii"))
     if isinstance(name, str):
-        if str_identifier_regex.fullmatch(name) is not None:
+        if is_valid_identifier:
             formatted_name = name
         else:
             formatted_name = "`" + name.replace("`", "``") + "`"
     else:
         assert isinstance(name, bytes)
-        if bytes_identifier_regex.fullmatch(name) is not None:
+        if is_valid_identifier:
             formatted_name = name
         else:
             formatted_name = b"`" + name.replace(b"`", b"``") + b"`"
@@ -81,6 +75,62 @@ def _quote_value(value):
     return quoted_value
 
 
+def _check_name(name):
+    """Ensures the variable name is a valid identifier
+
+    Plain string or bytes are both accepted as input.
+    Please note the Khiops core forbids a name
+        - with a length outside the [1,128] interval
+        - containing a simple (Unix) carriage-return (\n)
+        - with leading and trailing spaces
+            (\s in Perl-Compatible-Regular-Expressions syntax).
+    This function must check at least these constraints.
+    """
+    # Check that the type of name is string or bytes
+    if not is_string_like(name):
+        raise TypeError(type_error_message("name", name, "string-like"))
+
+    # Accentuated characters (between 128 and 255) are also accepted
+    # BUT NOT the greek ones
+    extended_identifier_pattern = r"^[a-zA-Z_\x80-\xFF][a-zA-Z0-9_\x80-\xFF]{0,127}$"
+    str_identifier_regex = re.compile(extended_identifier_pattern)
+    bytes_identifier_regex = re.compile(
+        bytes(extended_identifier_pattern, encoding="ascii")
+    )
+    if isinstance(name, str):
+        return str_identifier_regex.match(name) is not None
+    else:
+        assert isinstance(name, bytes)
+        return bytes_identifier_regex.match(name) is not None
+
+
+def _is_valid_type(type_str):
+    """Checks whether the type is known"""
+    return (
+        _is_native_type(type_str)
+        or _is_object_type(type_str)
+        or type_str in ["TextList", "Structure"]
+    )  # internal types
+
+
+def _is_native_type(type_str):
+    """Checks whether the type is native (not internal or relational)"""
+    return type_str in [
+        "Categorical",
+        "Numerical",
+        "Time",
+        "Date",
+        "Timestamp",
+        "TimestampTZ",
+        "Text",
+    ]
+
+
+def _is_object_type(type_str):
+    """Checks whether the type is an object one (relational)"""
+    return type_str in ["Entity", "Table"]
+
+
 class DictionaryDomain(KhiopsJSONObject):
     """Main class containing the information of a Khiops dictionary file
 
@@ -769,6 +819,137 @@ def add_variable(self, variable):
         self.variables.append(variable)
         self._variables_by_name[variable.name] = variable
 
+    def add_variable_from_spec(
+        self,
+        name,
+        type,
+        label=None,
+        used=True,
+        object_type=None,
+        structure_type=None,
+        rule=None,
+        meta_data=None,
+    ):
+        """Adds a variable to this dictionary using a complete specification
+
+        Parameters
+        ----------
+        name : str
+            Variable name
+        type : str
+            Variable type, See `Variable`
+        label : str, optional
+            Label of the variable.
+        used : bool, default ``True``
+            Usage status of the variable.
+        object_type : str, optional
+            Object type. Ignored if variable type not in ["Entity", "Table"]
+        structure_type : str, optional
+            Structure type. Ignored if variable type is not "Structure"
+        rule : str, optional
+            Variable rule (in verbatim).
+        meta_data : dict, optional
+            A Python dictionary which holds the metadata specification
+            with the following keys:
+            - keys : list, default []
+                list of meta-data keys
+            - values : list, default []
+                list of meta-data values.
+                The values can be str, bool, float or int.
+
+        Raises
+        ------
+        `ValueError`
+            - If the variable name is empty or does not comply
+              with the formatting constraints.
+            - If there is already a variable with the same name.
+            - If the given variable type is unknown.
+            - If a native type is given 'object_type' or 'structure_type'
+            - If the 'meta_data' is not a dictionary
+        """
+        # Values and Types checks
+        if not name:
+            raise ValueError(
+                "Cannot add to dictionary unnamed variable " f"(name = '{name}')"
+            )
+        if not _check_name(name):
+            raise ValueError(
+                f"New variable name '{name}' cannot be accepted "
+                "(invalid length or characters)"
+            )
+        if name in self._variables_by_name:
+            raise ValueError(f"Dictionary already has a variable named '{name}'")
+        if not _is_valid_type(type):
+            raise ValueError(f"Invalid type '{type}'")
+        if _is_native_type(type):
+            if object_type or structure_type:
+                raise ValueError(
+                    f"Native type '{type}' "
+                    "cannot have 'object_type' or 'structure_type'"
+                )
+        if _is_object_type(type) and object_type is None:
+            raise ValueError(f"'object_type' must be provided for type '{type}'")
+        if meta_data is not None:
+            if not is_dict_like(meta_data):
+                raise TypeError(type_error_message("meta_data", meta_data, "dict-like"))
+            if "keys" not in meta_data or "values" not in meta_data:
+                raise ValueError(
+                    "'meta_data' does not contain "
+                    "the mandatory keys 'keys' and 'values'"
+                )
+            if not is_list_like(meta_data["keys"]):
+                raise TypeError(
+                    type_error_message(
+                        "meta_data['keys']", meta_data["keys"], "list-like"
+                    )
+                )
+            if not is_list_like(meta_data["values"]):
+                raise TypeError(
+                    type_error_message(
+                        "meta_data['values']", meta_data["values"], "list-like"
+                    )
+                )
+            if len(meta_data["keys"]) != len(meta_data["values"]):
+                raise ValueError(
+                    "'meta_data' keys and values " "do not have the same size"
+                )
+        if label is not None:
+            if not is_string_like(label):
+                raise TypeError(type_error_message("label", label, "string-like"))
+        if object_type is not None:
+            if not is_string_like(object_type):
+                raise TypeError(
+                    type_error_message("object_type", object_type, "string-like")
+                )
+        if structure_type is not None:
+            if not is_string_like(structure_type):
+                raise TypeError(
+                    type_error_message("structure_type", structure_type, "string-like")
+                )
+        if rule is not None:
+            if not is_string_like(rule):
+                raise TypeError(
+                    type_error_message("rule", structure_type, "string-like")
+                )
+
+        # Variable initialization
+        variable = Variable()
+        variable.name = name
+        variable.type = type
+        variable.used = used
+        if meta_data is not None:
+            for key, value in zip(meta_data["keys"], meta_data["values"]):
+                variable.meta_data.add_value(key, value)
+        if label is not None:
+            variable.label = label
+        if object_type is not None:
+            variable.object_type = object_type
+        if structure_type is not None:
+            variable.structure_type = structure_type
+        if rule is not None:
+            variable.rule = Rule(verbatim=rule)
+        self.add_variable(variable)
+
     def remove_variable(self, variable_name):
         """Removes the specified variable from this dictionary
 
@@ -1058,7 +1239,7 @@ def __init__(self, json_data=None):
         self.type = json_data.get("type")
 
         # Initialize complement of the type
-        if self.type in ("Entity", "Table"):
+        if _is_object_type(self.type):
             self.object_type = json_data.get("objectType")
         elif self.type == "Structure":
             self.structure_type = json_data.get("structureType")
@@ -1072,7 +1253,7 @@ def __init__(self, json_data=None):
             self.meta_data = MetaData(json_meta_data)
 
     def __repr__(self):
-        """Returns a human readable string representation"""
+        """Returns a human-readable string representation"""
         return f"Variable ({self.name})"
 
     def __str__(self):
@@ -1179,7 +1360,7 @@ def full_type(self):
             basic.
         """
         full_type = self.type
-        if self.type in ("Entity", "Table"):
+        if _is_object_type(self.type):
             full_type += f"({self.object_type})"
         elif self.type == "Structure":
             full_type += f"({self.structure_type})"