set the parameter of seq2vec 0.4.0

Author: plliao

seq2vec/model/seq2vec_C2R_char.py

@@ -1,18 +1,16 @@
 """Sequence-to-Sequence word2vec."""
-
 import keras.models
 from keras.optimizers import RMSprop
 from keras.layers import Input, Reshape
 from keras.layers.core import Dense
 from keras.layers.wrappers import TimeDistributed
-from keras.layers import Conv2D
+from keras.layers import Conv2D, LSTM
 from keras.layers.pooling import MaxPool2D
 from keras.models import Model
-from keras import regularizers
 
 from yklz import MaskConv, ConvEncoder, MaskConvNet
 from yklz import MaskToSeq, MaskPooling, Pick
-from yklz import RNNDecoder, LSTMPeephole, RNNCell
+from yklz import RNNDecoder, RNNCell
 
 from seq2vec.transformer import CharEmbeddingOneHotTransformer
 from seq2vec.transformer import WordEmbeddingTransformer
@@ -40,7 +38,7 @@ def __init__(
             word2vec_model,
             max_index=10000,
             max_length=10,
-            embedding_size=300,
+            char_embedding_size=300,
             learning_rate=0.0001,
             conv_size=5,
             channel_size=10,
@@ -57,24 +55,32 @@ def __init__(
             word2vec_model,
             max_length,
         )
-        self.embedding_size = embedding_size
+        self.char_embedding_size = char_embedding_size
         self.max_index = max_index
         self.conv_size = conv_size
         self.channel_size = channel_size
         self.encoding_size = (
-            self.embedding_size // self.conv_size * self.channel_size
+            self.char_embedding_size // self.conv_size * self.channel_size
         )
 
         super(Seq2VecC2RChar, self).__init__(
             max_length,
             latent_size,
             learning_rate
         )
+        self.custom_objects['RNNDecoder'] = RNNDecoder
+        self.custom_objects['MaskPooling'] = MaskPooling
+        self.custom_objects['MaskToSeq'] = MaskToSeq
+        self.custom_objects['MaskConv'] = MaskConv
+        self.custom_objects['MaskConvNet'] = MaskConvNet
+        self.custom_objects['ConvEncoder'] = ConvEncoder
+        self.custom_objects['RNNCell'] = RNNCell
+        self.custom_objects['Pick'] = Pick
 
     def create_model(
             self,
             rho=0.9,
-            decay=0.01,
+            decay=0.0,
         ):
 
         inputs = Input(
@@ -85,16 +91,15 @@ def create_model(
         )
         char_embedding = TimeDistributed(
             Dense(
-                self.embedding_size,
+                self.char_embedding_size,
                 use_bias=False,
-                kernel_regularizer=regularizers.l2(0.001),
                 activation='tanh'
             )
         )(inputs)
 
         char_embedding = Reshape((
             self.max_length,
-            self.embedding_size,
+            self.char_embedding_size,
             1
         ))(char_embedding)
         masked_embedding = MaskConv(0.0)(char_embedding)
@@ -109,19 +114,18 @@ def create_model(
                 (2, self.conv_size),
                 strides=(1, self.conv_size),
                 activation='tanh',
-                padding='valid',
+                padding='same',
                 use_bias=False,
-                kernel_regularizer=regularizers.l2(0.001)
             )
         )(masked_embedding)
 
-        final_window_size = self.max_length - 1
-        final_feature_size = self.channel_size * self.embedding_size // self.conv_size
-
         mask_feature = MaskPooling(
             MaxPool2D(
-                (final_window_size, 1),
-                padding='valid'
+                (
+                    self.max_length,
+                    1
+                ),
+                padding='same'
             ),
             pool_mode='max'
         )(char_feature)
@@ -132,28 +136,25 @@ def create_model(
 
         dense_input = RNNDecoder(
             RNNCell(
-                LSTMPeephole(
+                LSTM(
                     units=self.latent_size,
                     return_sequences=True,
                     implementation=2,
                     unroll=False,
-                    dropout=0.1,
-                    recurrent_dropout=0.1,
-                    kernel_regularizer=regularizers.l2(0.001),
-                    recurrent_regularizer=regularizers.l2(0.001),
+                    dropout=0.,
+                    recurrent_dropout=0.,
                 ),
                 Dense(
-                    units=final_feature_size,
+                    units=self.encoding_size,
                     activation='tanh'
                 ),
-                dense_dropout=0.1
+                dense_dropout=0.
             )
         )(encoded_feature)
 
         outputs = TimeDistributed(
             Dense(
                 self.word_embedding_size,
-                kernel_regularizer=regularizers.l2(0.001),
                 activation='tanh'
             )
         )(dense_input)
@@ -170,32 +171,18 @@ def create_model(
         model.compile(loss='cosine_proximity', optimizer=optimizer)
         return model, encoder
 
-    def load_customed_model(self, file_path):
-        return keras.models.load_model(
-            file_path, custom_objects={
-                'RNNDecoder': RNNDecoder,
-                'MaskPooling': MaskPooling,
-                'MaskToSeq': MaskToSeq,
-                'MaskConv': MaskConv,
-                'MaskConvNet': MaskConvNet,
-                'ConvEncoder': ConvEncoder,
-                'LSTMPeephole':LSTMPeephole,
-                'RNNCell':RNNCell,
-                'Pick':Pick
-            }
-        )
-
     def load_model(self, file_path):
         self.model = self.load_customed_model(file_path)
         picked = Pick()(self.model.get_layer(index=7).output)
         self.encoder = Model(
             self.model.input,
             picked
         )
-        self.embedding_size = self.model.get_layer(index=1).output_shape[2]
+        self.char_embedding_size = self.model.get_layer(index=1).output_shape[2]
         self.max_length = self.model.get_layer(index=0).output_shape[1]
         self.max_index = self.model.input_shape[2]
-        self.conv_size = self.embedding_size // self.model.get_layer(index=4).output_shape[2]
+        self.conv_size = self.char_embedding_size \
+                         // self.model.get_layer(index=4).output_shape[2]
         self.channel_size = self.model.get_layer(index=4).output_shape[3]
         self.encoding_size = self.encoder.output_shape[1]
         self.latent_size = self.model.get_layer(index=8).layer.recurrent_layer.units

seq2vec/model/seq2vec_C2R_word.py

@@ -1,15 +1,14 @@
 """Sequence-to-Sequence word2vec."""
 import keras.models
 from keras.optimizers import RMSprop
-from keras.layers import Input, Conv3D
-from keras.layers.core import Dense
+from keras.layers import Input, Conv3D, LSTM
+from keras.layers.core import Dense, Dropout
 from keras.layers.wrappers import TimeDistributed
 from keras.layers.pooling import MaxPooling3D
 from keras.models import Model
-from keras import regularizers
 
 from yklz import MaskConv, MaskConvNet, MaskPooling, ConvEncoder
-from yklz import MaskToSeq, RNNDecoder, RNNCell, LSTMPeephole, Pick
+from yklz import MaskToSeq, RNNDecoder, RNNCell, Pick
 from seq2vec.transformer import WordEmbeddingConv3DTransformer
 from seq2vec.transformer import WordEmbeddingTransformer
 from seq2vec.model import TrainableSeq2VecBase
@@ -50,18 +49,26 @@ def __init__(
             word2vec_model,
             max_length
         )
-        self.embedding_size = word2vec_model.get_size()
+        self.word_embedding_size = word2vec_model.get_size()
         self.conv_size = conv_size
         self.channel_size = channel_size
         self.encoding_size = (
-            self.embedding_size // self.conv_size * self.channel_size
+            self.word_embedding_size // self.conv_size * self.channel_size
         )
 
         super(Seq2VecC2RWord, self).__init__(
             max_length,
             latent_size,
             learning_rate
         )
+        self.custom_objects['RNNDecoder'] = RNNDecoder
+        self.custom_objects['MaskPooling'] = MaskPooling
+        self.custom_objects['MaskToSeq'] = MaskToSeq
+        self.custom_objects['MaskConv'] = MaskConv
+        self.custom_objects['MaskConvNet'] = MaskConvNet
+        self.custom_objects['ConvEncoder'] = ConvEncoder
+        self.custom_objects['RNNCell'] = RNNCell
+        self.custom_objects['Pick'] = Pick
 
     def create_model(
             self,
@@ -73,15 +80,11 @@ def create_model(
             shape=(
                 self.max_length,
                 self.max_length,
-                self.embedding_size,
+                self.word_embedding_size,
                 1
             )
         )
 
-        final_window_size = self.max_length - 1
-        final_feature_size = self.embedding_size // self.conv_size * self.channel_size
-        final_feature_window_size = 1
-
         masked_inputs = MaskConv(0.0)(inputs)
         masked_seqs = MaskToSeq(
             MaskConv(0.0),
@@ -94,20 +97,19 @@ def create_model(
                 (2, 2, self.conv_size),
                 strides=(1, 1, self.conv_size),
                 activation='tanh',
-                padding='valid',
+                padding='same',
                 use_bias=False,
-                kernel_regularizer=regularizers.l2(0.001)
             )
         )(masked_inputs)
 
         pooling = MaskPooling(
             MaxPooling3D(
                 (
-                    final_window_size,
-                    final_window_size,
-                    final_feature_window_size
+                    self.max_length,
+                    self.max_length,
+                    1
                 ),
-                padding='valid'
+                padding='same'
             ),
             pool_mode='max'
         )(conv)
@@ -118,28 +120,25 @@ def create_model(
 
         decoded = RNNDecoder(
             RNNCell(
-                LSTMPeephole(
+                LSTM(
                     self.latent_size,
                     return_sequences=True,
                     implementation=2,
                     unroll=False,
-                    dropout=0.1,
-                    recurrent_dropout=0.1,
-                    kernel_regularizer=regularizers.l2(0.001),
-                    recurrent_regularizer=regularizers.l2(0.001)
+                    dropout=0.,
+                    recurrent_dropout=0.,
                 ),
                 Dense(
-                    units=final_feature_size,
+                    units=self.encoding_size,
                     activation='tanh'
                 ),
-                dense_dropout=0.1
+                dense_dropout=0.
             )
         )(encoded)
 
         outputs = TimeDistributed(
             Dense(
-                self.embedding_size,
-                kernel_regularizer=regularizers.l2(0.001),
+                self.word_embedding_size,
                 activation='tanh'
             )
         )(decoded)
@@ -156,29 +155,14 @@ def create_model(
         model.compile(loss='cosine_proximity', optimizer=optimizer)
         return model, encoder
 
-    def load_customed_model(self, file_path):
-        return keras.models.load_model(
-            file_path, custom_objects={
-                'RNNDecoder': RNNDecoder,
-                'MaskPooling': MaskPooling,
-                'MaskToSeq': MaskToSeq,
-                'MaskConv': MaskConv,
-                'MaskConvNet': MaskConvNet,
-                'ConvEncoder': ConvEncoder,
-                'LSTMPeephole':LSTMPeephole,
-                'RNNCell':RNNCell,
-                'Pick':Pick
-            }
-        )
-
     def load_model(self, file_path):
         self.model = self.load_customed_model(file_path)
         picked = Pick()(self.model.get_layer(index=5).output)
         self.encoder = Model(
             self.model.input,
             picked
         )
-        self.embedding_size = self.model.input_shape[3]
+        self.word_embedding_size = self.model.input_shape[3]
         self.max_length = self.model.input_shape[1]
         self.conv_size = self.model.get_layer(index=2).layer.kernel_size[2]
         self.latent_size = self.model.get_layer(index=6).layer.recurrent_layer.units