fix: Fix error caused when input data is mixed with datetime and string types (#7)

* fix: Fix error caused when input data is mixed with datetime and string types

Signed-off-by: tashiro akira <fj1755jk@fujitsu.com>

* fix: Fix error caused when input data is mixed with datetime and string types

Signed-off-by: tashiro akira <fj1755jk@fujitsu.com>

* fix: Fix error caused when input data is mixed with datetime and string types

Signed-off-by: tashiro akira <fj1755jk@fujitsu.com>

* fix:Modifying Source Code Formatting

Signed-off-by: tashiro akira <fj1755jk@fujitsu.com>

* fix:Reflect Review

Signed-off-by: tashiro akira <fj1755jk@fujitsu.com>

* fix: Remove Unnecessary Imports

Signed-off-by: tashiro akira <fj1755jk@fujitsu.com>

* fix: Reflect the point

Signed-off-by: tashiro akira <fj1755jk@fujitsu.com>

---------

Signed-off-by: tashiro akira <fj1755jk@fujitsu.com>

Author: tashiro-akira

sapientml_preprocess/generator.py

@@ -265,15 +265,36 @@ def generate_code(self, dataset: Dataset, task: Task) -> Tuple[Dataset, Code]:
             only_str = col + "__str"
             only_num = col + "__num"
             df[only_str] = np.where(pd.to_numeric(df[col], errors="coerce").isnull(), df[col], np.nan)
+            df[only_str] = np.where(df[only_str].notnull(), df[only_str].astype(str), np.nan)
             # without .astype(float), cannot recongnize as `int` or `float`, leading to generate inappropriate code snippet
             df[only_num] = np.where(pd.to_numeric(df[col], errors="coerce").isnull(), np.nan, df[col]).astype(float)
             df = df.drop(col, axis=1)
         if cols_numeric_and_string:
             tpl = template_env.get_template("handle_mixed_typed_columns.py.jinja")
-            code.validation += _render(tpl, training=True, test=True, cols_numeric_and_string=cols_numeric_and_string)
-            code.test += _render(tpl, training=True, test=True, cols_numeric_and_string=cols_numeric_and_string)
-            code.train += _render(tpl, training=True, test=False, cols_numeric_and_string=cols_numeric_and_string)
-            code.predict += _render(tpl, training=False, test=True, cols_numeric_and_string=cols_numeric_and_string)
+            code.validation += _render(
+                tpl,
+                training=True,
+                test=True,
+                cols_numeric_and_string=cols_numeric_and_string
+            )
+            code.test += _render(
+                tpl,
+                training=True,
+                test=True,
+                cols_numeric_and_string=cols_numeric_and_string
+            )
+            code.train += _render(
+                tpl,
+                training=True,
+                test=False,
+                cols_numeric_and_string=cols_numeric_and_string
+            )
+            code.predict += _render(
+                tpl,
+                training=False,
+                test=True,
+                cols_numeric_and_string=cols_numeric_and_string
+            )
 
         # meta features must be calculated after replacing inf with nan,
         # becuase the replaced nan must be preprocessed in the generated code.

sapientml_preprocess/templates/handle_mixed_typed_columns.py.jinja

@@ -4,11 +4,13 @@ cols_numeric_and_string = {{ cols_numeric_and_string}}
 for col in cols_numeric_and_string:
 {% if training %}
     train_dataset[col + '__str'] = np.where(pd.to_numeric(train_dataset[col], errors='coerce').isnull(), train_dataset[col], np.nan)
+    train_dataset[col + '__str'] = np.where(train_dataset[col + '__str'].notnull(), train_dataset[col + '__str'].astype(str), np.nan)
     train_dataset[col + '__num'] = np.where(pd.to_numeric(train_dataset[col], errors='coerce').isnull(), np.nan, train_dataset[col]).astype(float)
     train_dataset = train_dataset.drop(col, axis=1)
 {% endif %}
 {% if test %}
     test_dataset[col + '__str'] = np.where(pd.to_numeric(test_dataset[col], errors='coerce').isnull(), test_dataset[col], np.nan)
+    test_dataset[col + '__str'] = np.where(test_dataset[col + '__str'].notnull(), test_dataset[col + '__str'].astype(str), np.nan)
     test_dataset[col + '__num'] = np.where(pd.to_numeric(test_dataset[col], errors='coerce').isnull(), np.nan, test_dataset[col]).astype(float)
     test_dataset = test_dataset.drop(col, axis=1)
 {% endif %}