Merge branch 'master' into dependabot/pip/notebook-6.1.5

Author: maxsch3

keras_batchflow/base/batch_generators/batch_generator.py

@@ -35,18 +35,25 @@ class BatchGenerator:
     - **shuffle** - (optional) *bool*, if true, the input dataframe is shuffled before each new epoch.
         **Default: False**
     - **train_mode** - (optional) *bool*. If true, both X and Y are returned, otherwise only X is returned
+    - **encoder_adapter** - (optional) *str* or a single instance of a class derived from
+        keras_batchflow.base.batch_shapers.IEncoderAdaptor class. String values supported: 'numpy' and 'pandas'. If
+        not provided, 'numpy' is used. This parameter sets format that encoders are using. Sklearn encoders are
+        created for numpy arrays hence the default value is numpy. If your encoders require pandas format, use
+        'pandas'. Alternatively, if your encoders need some special format, create your instance derived from
+        IEncoderAdaptor class
     """
 
     def __init__(self, data: pd.DataFrame, x_structure, y_structure=None,
-                 batch_transforms=None, batch_size=32, shuffle=True, train_mode=True):
+                 batch_transforms=None, batch_size=32, shuffle=True, train_mode=True, encoder_adaptor=None):
         self.data = data
         self.batch_size = batch_size
         self.shuffle = shuffle
         self.train_mode = train_mode
         self.__check_batch_transformers(batch_transforms)
         self.batch_transforms = batch_transforms
         self.batch_shaper = BatchShaper(x_structure, y_structure,
-                                        data_sample=self._apply_batch_transforms(data.iloc[:min(data.shape[0], 10)]))
+                                        data_sample=self._apply_batch_transforms(data.iloc[:min(data.shape[0], 10)]),
+                                        encoder_adaptor=encoder_adaptor)
         self.indices = np.arange(self.data.shape[0])
         self.on_epoch_end()
 

keras_batchflow/base/batch_shapers/__init__.py

@@ -1 +1,4 @@
 from .batch_shaper import BatchShaper
+from .encoder_adaptor import IEncoderAdaptor
+from .numpy_encoder_adaptor import NumpyEncoderAdaptor
+from .pandas_encoder_adaptor import PandasEncoderAdaptor

keras_batchflow/base/batch_shapers/batch_shaper.py

@@ -16,9 +16,12 @@ class BatchShaper:
     with train/test splitted datasets
     """
 
-    def __init__(self, x_structure, y_structure=None, data_sample=None, multiindex_xy_keys=('x', 'y')):
+    def __init__(self, x_structure, y_structure=None, data_sample=None, multiindex_xy_keys=None,
+                 encoder_adaptor=None):
+        multiindex_xy_keys = ('x', 'y') if multiindex_xy_keys is None else multiindex_xy_keys
         self._validate_multiindex_xy_keys(x_structure, y_structure, multiindex_xy_keys)
         self.multiindex_xy_keys = multiindex_xy_keys
+        self._encoder_adaptor = encoder_adaptor
         data_sample_x, data_sample_y = self._get_data_xy(data_sample)
         self.x_structure = self._create_shapers(structure=x_structure, data_sample=data_sample_x)
         self.y_structure = self._create_shapers(structure=y_structure, data_sample=data_sample_y)
@@ -63,7 +66,7 @@ def _create_shapers(self, structure, data_sample):
 
     def _create_shaper_func(self, data, leaf, **kwargs):
         self._check_structure_leaf(leaf)
-        return VarShaper(var_name=leaf[0], encoder=leaf[1], data_sample=data)
+        return VarShaper(var_name=leaf[0], encoder=leaf[1], data_sample=data, encoder_adaptor=self._encoder_adaptor)
 
     def _walk(self, data: pd.DataFrame, func, **kwargs):
         data_x, data_y = self._get_data_xy(data)

keras_batchflow/base/batch_shapers/encoder_adaptor.py

@@ -0,0 +1,33 @@
+from abc import abstractmethod
+import pandas as pd
+
+
+class IEncoderAdaptor:
+
+    """
+    This class is used for converting data between source data in pandas dataframe and encoders, which might accept
+    different format types, e.g. numpy array or pandas series
+
+    This is an interface class for defining 2 standard adapters for numpy-based and pandas-based encoders. It can
+    also be used for defining your own class if needed.
+    """
+
+    @abstractmethod
+    def transform(self, x: pd.Series):
+        """
+        This method converts data before sending to an encoder. Define format conversion here
+        :param x:
+        :return:
+        """
+        pass
+
+    @abstractmethod
+    def inverse_transform(self, x, dtype=None) -> pd.Series:
+        """
+        This method coverts data received from encoder back into pandas Series
+        :param x: data in the format of encoder
+        :param dtype: optional, target dtype for data in pandas Series object created. Normally this is an original dtype
+        from original data. If not provided, the data type will be inferred by pandas
+        :return: a pandas series object
+        """
+        pass

keras_batchflow/base/batch_shapers/numpy_encoder_adaptor.py

@@ -0,0 +1,39 @@
+import pandas as pd
+import numpy as np
+
+from .encoder_adaptor import IEncoderAdaptor
+
+
+class NumpyEncoderAdaptor(IEncoderAdaptor):
+
+    """
+    This is an adapter that converts data for Numpy based encoders.
+    """
+
+    def transform(self, x: pd.Series):
+        """
+        This method converts pandas Series object to a numpy array. It uses to_numpy method rather than values
+        property because some Pandas data types do not exist in numpy (e.g. IntegerArray) and by using to_numpy,
+        we involve pandas internal conversion
+        :param x:
+        :return:
+        """
+        return x.to_numpy()
+
+    def inverse_transform(self, x, dtype=None) -> pd.Series:
+        """
+        Both input and output are pandas Series here. THis method will only check that encoder indeed returns
+        pandas Series and will make sure the data type is correct
+        :param x:
+        :param dtype:
+        :return:
+        """
+        if not isinstance(x, np.ndarray):
+            raise TypeError(f"Error: the encoder is supposed to return numpy array, got {type(x)}")
+        if x.ndim > 1:
+            x = np.squeeze(x)
+            if x.ndim > 1:
+                raise ValueError(f"Error: the encoder is supposed to return 1D data. Got {x.ndim}D even after "
+                                 f"squeezing")
+        x = pd.Series(x, dtype=dtype)
+        return x

keras_batchflow/base/batch_shapers/pandas_encoder_adaptor.py

@@ -0,0 +1,33 @@
+import pandas as pd
+
+from .encoder_adaptor import IEncoderAdaptor
+
+
+class PandasEncoderAdaptor(IEncoderAdaptor):
+
+    """
+    This is an adapter that converts data for Pandas based encoders.
+    """
+
+    def transform(self, x: pd.Series):
+        """
+        Because both input and output data is a pandas Series there will be no conversion
+        :param x:
+        :return:
+        """
+        return x
+
+    def inverse_transform(self, x, dtype=None) -> pd.Series:
+        """
+        Both input and output are pandas Series here. THis method will only check that encoder indeed returns
+        pandas Series and will make sure the data type is correct
+        :param x:
+        :param dtype:
+        :return:
+        """
+        if not isinstance(x, pd.Series):
+            raise TypeError(f"Error: the encoder is supposed to return Pandas Series object, got {type(x)}")
+        if dtype is not None:
+            if x.dtype != dtype:
+                x = x.astype(dtype)
+        return x

keras_batchflow/base/batch_shapers/var_shaper.py

@@ -2,6 +2,10 @@
 import pandas as pd
 from numbers import Number
 
+from .encoder_adaptor import IEncoderAdaptor
+from .numpy_encoder_adaptor import NumpyEncoderAdaptor
+from .pandas_encoder_adaptor import PandasEncoderAdaptor
+
 
 class VarShaper:
 
@@ -20,10 +24,11 @@ class VarShaper:
 
     _dummy_constant_counter = 0
 
-    def __init__(self, var_name, encoder, data_sample=None):
+    def __init__(self, var_name, encoder, data_sample=None, encoder_adaptor=None):
         self._var_name = var_name
         # _name will be included in metadata for using in ML models, e.g. for naming input layers in Keras
         self._name = var_name
+        self._encoder_adaptor = self._build_encoder_adaptor(encoder_adaptor)
         self._encoder = encoder
         self._class = self._self_classify(var_name, encoder)
         self._decoded_dtype, self._dtype = self._get_dtypes(data_sample)
@@ -34,6 +39,24 @@ def __init__(self, var_name, encoder, data_sample=None):
         self._shape = self._get_shape(var_name, encoder, data_sample)
         self._n_classes = self._get_n_classes(encoder)
 
+    def _build_encoder_adaptor(self, encoder_adaptor):
+        """
+        This method makes encoder adaptor class that utilises polymorphism to accommodate encoders that require
+        different data types. The encoder adaptor instance takes care of
+        :param encoder_adaptor: str ('numpy' or 'pandas') or a custom class derived from IEncoderAdaptor
+        :return: instance of IEncoderAdaptor
+        """
+        if (encoder_adaptor == 'numpy') or (encoder_adaptor is None):
+            adaptor = NumpyEncoderAdaptor()
+        elif encoder_adaptor == 'pandas':
+            adaptor = PandasEncoderAdaptor()
+        elif isinstance(encoder_adaptor, IEncoderAdaptor):
+            adaptor = encoder_adaptor
+        else:
+            raise TypeError(f"Error: The encoder adaptor must be a string ('numpy' or 'pandas') or an instance of a "
+                            f"custom class derived from IEncoderAdaptor")
+        return adaptor
+
     @staticmethod
     def _self_classify(var_name, encoder):
         """
@@ -58,6 +81,8 @@ def _self_classify(var_name, encoder):
                 return "direct"
             if not hasattr(encoder, "transform"):
                 raise ValueError(f"Error: encoder provided for column '{var_name}' has no 'transform' method")
+            if not hasattr(encoder, "inverse_transform"):
+                raise ValueError(f"Error: encoder provided for column '{var_name}' has no 'inverse_transform' method")
             return "encoder"
         else:
             raise ValueError(f"Error: variable name must be a str or None. Got {type(var_name)}")
@@ -155,20 +180,21 @@ def transform(self, data):
             # if not hasattr(self._encoder, 'transform'):
             #     raise ValueError(f"Error: encoders of class {type(self._encoder).__name__} provided in structure "
             #                      f"definition has no 'transform' method")
+            encoder_input = self._encoder_adaptor.transform(data[self._var_name])
             try:
-                x = getattr(self._encoder, 'transform')(data[self._var_name].values)
+                x = self._encoder.transform(encoder_input)
             except ValueError as e:
                 raise ValueError(f'Error: ValueError exception occured while calling '
                                  f'{type(self._encoder).__name__}.transform method. Most likely you used'
                                  f' 2D encoders. At the moment, only 1D transformers are supported. Please use 1D '
                                  f'variant or use wrapper. The error was: {e}')
-            except Exception as e:
-                raise RuntimeError(f'Error: unknown error while calling transform method of '
-                                   f'{type(self._encoder).__name__} class provided in structure. The error was: {e}')
+            # except Exception as e:
+            #     raise RuntimeError(f'Error: unknown error while calling transform method of '
+            #                        f'{type(self._encoder).__name__} class provided in structure. The error was: {e}')
         elif self._class == "constant":
             x = np.repeat(self._encoder, data.shape[0])
         elif self._class == "direct":
-            x = data[self._var_name].values
+            x = data[self._var_name].to_numpy()
         else:
             raise RuntimeError('Error: this should not have happened. Maybe it needs to be reported')
         # if self._dtype is None:
@@ -192,13 +218,13 @@ def inverse_transform(self, df, encoded_data):
             # changes. These cases have structure entry like this ('col_name', None)
             df[self._var_name] = pd.Series(np.squeeze(encoded_data), dtype=self._decoded_dtype)
         elif self._class == "encoder":
-            if not hasattr(self._encoder, "inverse_transform"):
-                raise ValueError(f"Error: encoder provided for column '{self._var_name}' has no 'inverse_transform' method")
-            if not hasattr(self._encoder, 'inverse_transform'):
-                raise ValueError('Error: the encoders {} used for column {} has no inverse_transform method'
-                                 .format(type(self._encoder).__name__, self._var_name))
-            it = self._encoder.inverse_transform(encoded_data)
-            df[self._var_name] = pd.Series(it, dtype=self._decoded_dtype)
+            # it has already been checked at init stage. It is redundant here
+            # if not hasattr(self._encoder, "inverse_transform"):
+            #     raise ValueError(f"Error: encoder provided for column '{self._var_name}' has no "
+            #     "'inverse_transform' method")
+            it = self._encoder_adaptor.inverse_transform(self._encoder.inverse_transform(encoded_data),
+                                                         dtype=self._decoded_dtype)
+            df[self._var_name] = it
 
     def _reshape(self, x: np.ndarray):
         if x.ndim == 1:

keras_batchflow/keras/batch_shapers/__init__.py

@@ -1 +1,4 @@
-from keras_batchflow.base.batch_shapers import BatchShaper
+from keras_batchflow.base.batch_shapers import BatchShaper
+from keras_batchflow.base.batch_shapers import IEncoderAdaptor
+from keras_batchflow.base.batch_shapers import NumpyEncoderAdaptor
+from keras_batchflow.base.batch_shapers import PandasEncoderAdaptor

keras_batchflow/tf/batch_shapers/__init__.py

@@ -1 +1,4 @@
-from keras_batchflow.base.batch_shapers import BatchShaper
+from keras_batchflow.base.batch_shapers import BatchShaper
+from keras_batchflow.base.batch_shapers import IEncoderAdaptor
+from keras_batchflow.base.batch_shapers import NumpyEncoderAdaptor
+from keras_batchflow.base.batch_shapers import PandasEncoderAdaptor

test/test_batch_generator.py

@@ -5,6 +5,9 @@
 from keras_batchflow.base.batch_transformers import BatchTransformer
 from sklearn.preprocessing import LabelEncoder, LabelBinarizer, OneHotEncoder
 
+from keras_batchflow.base.batch_shapers.numpy_encoder_adaptor import NumpyEncoderAdaptor
+from keras_batchflow.base.batch_shapers.pandas_encoder_adaptor import PandasEncoderAdaptor
+
 
 class TestBatchGenerator:
 
@@ -221,5 +224,27 @@ def test_shapes(self):
         assert sh[0][1] == (1,)
         assert sh[1] == (1,)
 
+    def test_encoder_adaptor(self):
+        """
+        This test only makes sure the adaptor parameter is passed correctly
+        :return:
+        """
+        bg = BatchGenerator(
+            self.df,
+            x_structure=('var1', self.lb),
+            y_structure=('label', self.le),
+            shuffle=False,
+            encoder_adaptor='numpy'
+        )
+        assert isinstance(bg.batch_shaper.x_structure._encoder_adaptor, NumpyEncoderAdaptor)
+        bg = BatchGenerator(
+            self.df,
+            x_structure=('var1', self.lb),
+            y_structure=('label', self.le),
+            shuffle=False,
+            encoder_adaptor='pandas'
+        )
+        assert isinstance(bg.batch_shaper.x_structure._encoder_adaptor, PandasEncoderAdaptor)
+
 if __name__ == '__main__':
     pytest.main([__file__])