Fix DeepFM dense-only loading, DNN save tracking, and FNN logits (#555)

- Avoid empty FM branches in dense-only DeepFM models
- Build DNN activation layers without mutating tracked lists
- Remove the WDL-style linear logit from FNN
- Replace test asserts flagged by Codacy
- Add targeted regression tests

Author: shenweichen

deepctr/estimator/models/fnn.py

@@ -8,7 +8,7 @@
 """
 import tensorflow as tf
 
-from ..feature_column import get_linear_logit, input_from_feature_columns
+from ..feature_column import input_from_feature_columns
 from ..utils import deepctr_model_fn, DNN_SCOPE_NAME, variable_scope
 from ...layers.core import DNN
 from ...layers.utils import combined_dnn_input
@@ -20,11 +20,11 @@ def FNNEstimator(linear_feature_columns, dnn_feature_columns, dnn_hidden_units=(
                  dnn_optimizer='Adagrad', training_chief_hooks=None):
     """Instantiates the Factorization-supported Neural Network architecture.
 
-    :param linear_feature_columns: An iterable containing all the features used by linear part of the model.
+    :param linear_feature_columns: An iterable containing features kept for API compatibility.
     :param dnn_feature_columns: An iterable containing all the features used by deep part of the model.
     :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of deep net
     :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
-    :param l2_reg_linear: float. L2 regularizer strength applied to linear weight
+    :param l2_reg_linear: float. Kept for API compatibility.
     :param l2_reg_dnn: float . L2 regularizer strength applied to DNN
     :param seed: integer ,to use as random seed.
     :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate.
@@ -47,8 +47,6 @@ def FNNEstimator(linear_feature_columns, dnn_feature_columns, dnn_hidden_units=(
     def _model_fn(features, labels, mode, config):
         train_flag = (mode == tf.estimator.ModeKeys.TRAIN)
 
-        linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear)
-
         with variable_scope(DNN_SCOPE_NAME):
             sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns,
                                                                                  l2_reg_embedding=l2_reg_embedding)
@@ -57,9 +55,7 @@ def _model_fn(features, labels, mode, config):
             dnn_logit = tf.keras.layers.Dense(
                 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(deep_out)
 
-        logits = linear_logits + dnn_logit
-
-        return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer,
+        return deepctr_model_fn(features, mode, dnn_logit, labels, task, linear_optimizer, dnn_optimizer,
                                 training_chief_hooks=training_chief_hooks)
 
     return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=config)

deepctr/layers/core.py

@@ -179,10 +179,10 @@ def build(self, input_shape):
         self.dropout_layers = [Dropout(self.dropout_rate, seed=self.seed + i) for i in
                                range(len(self.hidden_units))]
 
-        self.activation_layers = [activation_layer(self.activation) for _ in range(len(self.hidden_units))]
-
-        if self.output_activation:
-            self.activation_layers[-1] = activation_layer(self.output_activation)
+        self.activation_layers = [
+            activation_layer(
+                self.output_activation if i == len(self.hidden_units) - 1 and self.output_activation else self.activation)
+            for i in range(len(self.hidden_units))]
 
         super(DNN, self).build(input_shape)  # Be sure to call this somewhere!
 

deepctr/layers/utils.py

@@ -142,14 +142,14 @@ def build(self, input_shape):
                                         trainable=True)
         if self.mode == 1:
             self.kernel = self.add_weight(
-                'linear_kernel',
+                name='linear_kernel',
                 shape=[int(input_shape[-1]), 1],
                 initializer=glorot_normal(self.seed),
                 regularizer=l2(self.l2_reg),
                 trainable=True)
         elif self.mode == 2:
             self.kernel = self.add_weight(
-                'linear_kernel',
+                name='linear_kernel',
                 shape=[int(input_shape[1][-1]), 1],
                 initializer=glorot_normal(self.seed),
                 regularizer=l2(self.l2_reg),

deepctr/models/deepfm.py

@@ -50,15 +50,15 @@ def DeepFM(linear_feature_columns, dnn_feature_columns, fm_group=(DEFAULT_GROUP_
     group_embedding_dict, dense_value_list = input_from_feature_columns(features, dnn_feature_columns, l2_reg_embedding,
                                                                         seed, support_group=True)
 
-    fm_logit = add_func([FM()(concat_func(v, axis=1))
-                         for k, v in group_embedding_dict.items() if k in fm_group])
+    fm_logit_list = [FM()(concat_func(v, axis=1))
+                     for k, v in group_embedding_dict.items() if k in fm_group]
 
     dnn_input = combined_dnn_input(list(chain.from_iterable(
         group_embedding_dict.values())), dense_value_list)
     dnn_output = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed=seed)(dnn_input)
     dnn_logit = Dense(1, use_bias=False)(dnn_output)
 
-    final_logit = add_func([linear_logit, fm_logit, dnn_logit])
+    final_logit = add_func([linear_logit, dnn_logit] + fm_logit_list)
 
     output = PredictionLayer(task)(final_logit)
     model = Model(inputs=inputs_list, outputs=output)

deepctr/models/fnn.py

@@ -9,21 +9,21 @@
 from tensorflow.keras.models import Model
 from tensorflow.keras.layers import Dense
 
-from ..feature_column import build_input_features, get_linear_logit, input_from_feature_columns
+from ..feature_column import build_input_features, input_from_feature_columns
 from ..layers.core import PredictionLayer, DNN
-from ..layers.utils import add_func, combined_dnn_input
+from ..layers.utils import combined_dnn_input
 
 
 def FNN(linear_feature_columns, dnn_feature_columns, dnn_hidden_units=(256, 128, 64),
         l2_reg_embedding=1e-5, l2_reg_linear=1e-5, l2_reg_dnn=0, seed=1024, dnn_dropout=0,
         dnn_activation='relu', task='binary'):
     """Instantiates the Factorization-supported Neural Network architecture.
 
-    :param linear_feature_columns: An iterable containing all the features used by linear part of the model.
+    :param linear_feature_columns: An iterable containing features kept for API compatibility.
     :param dnn_feature_columns: An iterable containing all the features used by deep part of the model.
     :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of deep net
     :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
-    :param l2_reg_linear: float. L2 regularizer strength applied to linear weight
+    :param l2_reg_linear: float. Kept for API compatibility.
     :param l2_reg_dnn: float . L2 regularizer strength applied to DNN
     :param seed: integer ,to use as random seed.
     :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate.
@@ -36,18 +36,14 @@ def FNN(linear_feature_columns, dnn_feature_columns, dnn_hidden_units=(256, 128,
 
     inputs_list = list(features.values())
 
-    linear_logit = get_linear_logit(features, linear_feature_columns, seed=seed, prefix='linear',
-                                    l2_reg=l2_reg_linear)
-
     sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns,
                                                                          l2_reg_embedding, seed)
 
     dnn_input = combined_dnn_input(sparse_embedding_list, dense_value_list)
     deep_out = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, False, seed=seed)(dnn_input)
     dnn_logit = Dense(1, use_bias=False)(deep_out)
-    final_logit = add_func([dnn_logit, linear_logit])
 
-    output = PredictionLayer(task)(final_logit)
+    output = PredictionLayer(task)(dnn_logit)
 
     model = Model(inputs=inputs_list, outputs=output)
     return model

tests/layers/core_test.py

@@ -43,6 +43,15 @@ def test_DNN(hidden_units, use_bn):
                        BATCH_SIZE, EMBEDDING_SIZE))
 
 
+def test_DNN_output_activation():
+    with CustomObjectScope({'DNN': layers.DNN}):
+        x = tf.keras.layers.Input(shape=(EMBEDDING_SIZE,))
+        y = layers.DNN((10,), output_activation='sigmoid')(x)
+        model = tf.keras.models.Model(x, y)
+        if model.output_shape != (None, 10):
+            raise AssertionError("Unexpected DNN output shape")
+
+
 @pytest.mark.parametrize(
     'task,use_bias',
     [(task, use_bias)

tests/models/DeepFM_test.py

@@ -1,5 +1,11 @@
 import pytest
+import numpy as np
+import os
+import tempfile
+from tensorflow.keras.models import load_model, save_model
 
+from deepctr.feature_column import DenseFeat
+from deepctr.layers import custom_objects
 from deepctr.models import DeepFM
 from ..utils import check_model, get_test_data, SAMPLE_SIZE, get_test_data_estimator, check_estimator, TEST_Estimator
 
@@ -21,6 +27,27 @@ def test_DeepFM(hidden_size, sparse_feature_num):
     check_model(model, model_name, x, y)
 
 
+def test_DeepFM_dense_only_model_io():
+    sample_size = SAMPLE_SIZE
+    feature_columns = [DenseFeat('dense_feature_' + str(i), 1) for i in range(2)]
+    x = {fc.name: np.random.random(sample_size) for fc in feature_columns}
+    y = np.random.randint(0, 2, (sample_size, 1))
+
+    model = DeepFM(feature_columns, feature_columns, dnn_hidden_units=(4,), dnn_dropout=0)
+    model.compile('adam', 'binary_crossentropy',
+                  metrics=['binary_crossentropy'])
+    model.fit(x, y, batch_size=4, epochs=1, validation_split=0.5)
+
+    fd = tempfile.NamedTemporaryFile(suffix='.h5', delete=False)
+    model_path = fd.name
+    fd.close()
+    try:
+        save_model(model, model_path)
+        load_model(model_path, custom_objects)
+    finally:
+        os.remove(model_path)
+
+
 @pytest.mark.parametrize(
     'hidden_size,sparse_feature_num',
     [

tests/models/FNN_test.py

@@ -1,6 +1,7 @@
 import pytest
 import tensorflow as tf
 
+from deepctr.feature_column import DenseFeat, SparseFeat
 from deepctr.models import FNN
 from ..utils import check_model, get_test_data, SAMPLE_SIZE, get_test_data_estimator, check_estimator, TEST_Estimator
 
@@ -23,6 +24,16 @@ def test_FNN(sparse_feature_num, dense_feature_num):
     check_model(model, model_name, x, y)
 
 
+def test_FNN_does_not_add_wide_linear_logit():
+    feature_columns = [SparseFeat('sparse_feature', 4, embedding_dim=4),
+                       DenseFeat('dense_feature', 1)]
+
+    model = FNN(feature_columns, feature_columns, dnn_hidden_units=(4,), dnn_dropout=0)
+
+    if not all(layer.__class__.__name__ != 'Linear' for layer in model.layers):
+        raise AssertionError("FNN should not include a wide Linear layer")
+
+
 # @pytest.mark.parametrize(
 #     'sparse_feature_num,dense_feature_num',
 #     [(0, 1), (1, 0)